diff --git a/.github/actions/setup-protoc/action.yml b/.github/actions/setup-protoc/action.yml new file mode 100644 index 000000000..a99cf3320 --- /dev/null +++ b/.github/actions/setup-protoc/action.yml @@ -0,0 +1,80 @@ +name: Setup protoc +description: Install the protoc version pinned for wheel builds, and expose it for builds that install protoc themselves + +inputs: + install: + description: "Install protoc on the runner. Set to false when the build runs in a container (e.g. manylinux) and installs protoc there." + required: false + default: "true" + +outputs: + version: + description: "The pinned protoc version, e.g. 36.2" + value: ${{ steps.pin.outputs.version }} + linux-x86_64-sha256: + description: "SHA-256 of the protoc linux-x86_64 zip" + value: ${{ steps.pin.outputs.linux-x86_64-sha256 }} + linux-aarch_64-sha256: + description: "SHA-256 of the protoc linux-aarch_64 zip" + value: ${{ steps.pin.outputs.linux-aarch_64-sha256 }} + +runs: + using: composite + steps: + # protobuf does not publish checksum files for protoc. These are the digests + # GitHub records for the release assets; when bumping, copy them from + # `gh api repos/protocolbuffers/protobuf/releases/tags/v`. + - name: Set protoc pin + id: pin + shell: bash + run: | + { + echo "version=36.2" + echo "linux-x86_64-sha256=121f6c7afe1d4d0e3ea6aab9432038599250134cbf4474cb1167d2c7decd4278" + echo "linux-aarch_64-sha256=8b8f18bd2b30346efbc698dd5a73dd7c805f3ef8380f6dfc95c768f3f1852f6a" + echo "osx-universal_binary-sha256=83d0dc80672f53486f54299bca6177159f9cad1fd54d4d3143eeea53c4423e1b" + echo "win64-sha256=f0c128dc0d8492eceece83bb459a4c0e316764b929ffbf1aa416357fd644edd3" + } >> "$GITHUB_OUTPUT" + - name: Install protoc + if: ${{ inputs.install == 'true' }} + shell: bash + env: + PROTOC_VERSION: ${{ steps.pin.outputs.version }} + RUNNER_OS_NAME: ${{ runner.os }} + RUNNER_ARCH_NAME: ${{ runner.arch }} + LINUX_X86_64_SHA256: ${{ steps.pin.outputs.linux-x86_64-sha256 }} + LINUX_AARCH_64_SHA256: ${{ steps.pin.outputs.linux-aarch_64-sha256 }} + OSX_SHA256: ${{ steps.pin.outputs.osx-universal_binary-sha256 }} + WIN64_SHA256: ${{ steps.pin.outputs.win64-sha256 }} + run: | + set -euo pipefail + case "$RUNNER_OS_NAME/$RUNNER_ARCH_NAME" in + Linux/X64) platform="linux-x86_64"; sha256="$LINUX_X86_64_SHA256" ;; + Linux/ARM64) platform="linux-aarch_64"; sha256="$LINUX_AARCH_64_SHA256" ;; + macOS/*) platform="osx-universal_binary"; sha256="$OSX_SHA256" ;; + Windows/X64) platform="win64"; sha256="$WIN64_SHA256" ;; + *) echo "Unsupported runner: $RUNNER_OS_NAME/$RUNNER_ARCH_NAME" >&2; exit 1 ;; + esac + zip="$RUNNER_TEMP/protoc.zip" + dest="$RUNNER_TEMP/protoc" + curl -fsSL "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-${platform}.zip" -o "$zip" + # Hash stdin rather than a path: Windows paths contain backslashes, + # which make sha256sum escape its output. + if [ "$RUNNER_OS_NAME" = "macOS" ]; then + actual=$(shasum -a 256 < "$zip" | cut -d' ' -f1) + else + actual=$(sha256sum < "$zip" | cut -d' ' -f1) + fi + if [ "$actual" != "$sha256" ]; then + echo "protoc checksum mismatch: expected $sha256, got $actual" >&2 + exit 1 + fi + if [ "$RUNNER_OS_NAME" = "Windows" ]; then + # Git Bash on Windows runners does not reliably ship unzip. + 7z x -y -o"$dest" "$zip" > /dev/null + else + unzip -q -o "$zip" -d "$dest" + fi + rm "$zip" + echo "$dest/bin" >> "$GITHUB_PATH" + "$dest/bin/protoc" --version diff --git a/.github/workflows/build_linux_wheel/action.yml b/.github/workflows/build_linux_wheel/action.yml index 8fa5b1f79..44ce82ee5 100644 --- a/.github/workflows/build_linux_wheel/action.yml +++ b/.github/workflows/build_linux_wheel/action.yml @@ -29,6 +29,11 @@ inputs: runs: using: "composite" steps: + - name: Get protoc version + id: protoc + uses: ./.github/actions/setup-protoc + with: + install: "false" - name: CONFIRM ARM BUILD shell: bash env: @@ -54,13 +59,14 @@ runs: maturin-version: "1.12.4" command: build working-directory: python - docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}" + docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc -e PROTOC_VERSION=${{ steps.protoc.outputs.version }} -e PROTOC_SHA256=${{ steps.protoc.outputs.linux-x86_64-sha256 }} ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}" target: x86_64-unknown-linux-gnu manylinux: ${{ inputs.manylinux }} args: ${{ inputs.args }} before-script-linux: | set -e - curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-x86_64.zip -o /tmp/protoc.zip + curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip -o /tmp/protoc.zip + echo "${PROTOC_SHA256} /tmp/protoc.zip" | sha256sum -c - unzip /tmp/protoc.zip -d /usr/local rm /tmp/protoc.zip /usr/local/bin/protoc --version @@ -71,14 +77,15 @@ runs: maturin-version: "1.12.4" command: build working-directory: python - docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}" + docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc -e PROTOC_VERSION=${{ steps.protoc.outputs.version }} -e PROTOC_SHA256=${{ steps.protoc.outputs.linux-aarch_64-sha256 }} ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}" target: aarch64-unknown-linux-gnu manylinux: ${{ inputs.manylinux }} args: ${{ inputs.args }} before-script-linux: | set -e yum install -y clang - curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-aarch_64.zip -o /tmp/protoc.zip + curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-aarch_64.zip -o /tmp/protoc.zip + echo "${PROTOC_SHA256} /tmp/protoc.zip" | sha256sum -c - unzip /tmp/protoc.zip -d /usr/local rm /tmp/protoc.zip /usr/local/bin/protoc --version diff --git a/.github/workflows/build_mac_wheel/action.yml b/.github/workflows/build_mac_wheel/action.yml index 75d942c96..727cd190c 100644 --- a/.github/workflows/build_mac_wheel/action.yml +++ b/.github/workflows/build_mac_wheel/action.yml @@ -12,10 +12,7 @@ inputs: runs: using: "composite" steps: - - name: Install macos dependency - shell: bash - run: | - brew install protobuf + - uses: ./.github/actions/setup-protoc - name: Build wheel uses: PyO3/maturin-action@v1 with: diff --git a/.github/workflows/build_windows_wheel/action.yml b/.github/workflows/build_windows_wheel/action.yml index 18f39818f..954ee7d35 100644 --- a/.github/workflows/build_windows_wheel/action.yml +++ b/.github/workflows/build_windows_wheel/action.yml @@ -12,15 +12,7 @@ inputs: runs: using: "composite" steps: - - name: Install Protoc v21.12 - working-directory: C:\ - run: | - New-Item -Path 'C:\protoc' -ItemType Directory - Set-Location C:\protoc - Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip - 7z x protoc.zip - Add-Content $env:GITHUB_PATH "C:\protoc\bin" - shell: powershell + - uses: ./.github/actions/setup-protoc - name: Build wheel uses: PyO3/maturin-action@v1 with: diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index 4f883aec1..d16203979 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -11,6 +11,7 @@ on: - .github/workflows/build_linux_wheel/action.yml - .github/workflows/build_mac_wheel/action.yml - .github/workflows/build_windows_wheel/action.yml + - .github/actions/setup-protoc/action.yml - Cargo.toml # Change in dependency frequently breaks builds - Cargo.lock diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 3594bd826..3608c24ca 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -15,6 +15,7 @@ on: - .github/workflows/build_linux_wheel/** - .github/workflows/build_mac_wheel/** - .github/workflows/build_windows_wheel/** + - .github/actions/setup-protoc/** - .github/workflows/run_tests/** permissions: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a13086061..0aabd8fec 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -94,7 +94,7 @@ what the capability is, and what the embedded form does instead. A separate private repository supplies the fuller version of each, which replaces the page at the same path when the published site is assembled. -Two things to know before editing: +Four things to know before editing: **Code examples are not written into the pages.** They live in real tests under `docs/web-tests/{py,ts,rs}`, are extracted into `docs/web/snippets/`, and are @@ -116,6 +116,44 @@ Enterprise pages attach their additions to the right section, so they are assigned once and never regenerated. Leave an existing anchor alone even when you reword the heading above it; only new headings need a new one. +**A page can declare where it applies.** Its frontmatter says whether LanceDB +OSS and LanceDB Enterprise support the feature: use only `available` or +`unavailable`, and one sentence that explains its scope. Availability describes +product support, not whether a particular deployment has enabled or configured +the feature. A supported feature stays `available` when it requires setup, a +supported SDK or server version, or has differences between offerings. State +those requirements in the summary or the relevant section, including unsupported +subfeatures. Do not use `varies`, infer support from an unknown status, or mark an +absent capability available. Narrow a page to the capability it actually covers +when necessary. For example: + +```yaml +availability: + oss: unavailable + enterprise: available + summary: >- + Requests to a deployment carry an API key or an OAuth token. Embedded + LanceDB has no server to sign in to. +``` + +The assembled site renders this as a label above the page's content and as a +sidebar tag when only one offering has the feature. Do not also open the page +with a badge of its own, and keep a qualification that only one section needs +in that section's text. A `mint dev` preview of `docs/web` alone renders +neither the labels nor the tags. + +**A feature can be documented before its release.** List a draft page in +`docs/web/.mintignore`, keep it out of the navigation in `docs.json`, and do not +link to it from published pages. Wrap a draft section in an MDX comment, +`{/* ... */}`, whose first line names the release, such as +`Draft for the Feature X release.` Keep draft table rows and list items in a +comment below the table or list, since a comment between rows or items breaks +it. Drafts are hidden from readers but stay in the source, and availability does +not change what is published. At release, remove the comments and the +`.mintignore` entries, and restore the navigation and links. See +[Writing docs before release](https://github.com/lancedb/docs#writing-docs-before-release) +in the documentation repository's README for previews and release checks. + ## Release process For information on the release process, see: [release_process.md](release_process.md) diff --git a/Cargo.lock b/Cargo.lock index b05d85f5b..b4fbfa404 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3584,8 +3584,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow-array", "rand 0.9.5", @@ -5059,8 +5059,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arc-swap", "arrow", @@ -5133,7 +5133,7 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "14.0.0-beta.15" +version = "14.0.0-beta.18" dependencies = [ "arrow-array", "arrow-buffer", @@ -5155,7 +5155,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow-array", "arrow-buffer", @@ -5169,7 +5169,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow-array", "arrow-schema", @@ -5178,8 +5178,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrayref", "crunchy", @@ -5189,8 +5189,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow-array", "arrow-buffer", @@ -5227,8 +5227,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow", "arrow-array", @@ -5259,8 +5259,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow", "arrow-array", @@ -5277,8 +5277,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "proc-macro2", "quote", @@ -5287,8 +5287,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow-arith", "arrow-array", @@ -5321,8 +5321,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow-arith", "arrow-array", @@ -5354,8 +5354,8 @@ dependencies = [ [[package]] name = "lance-geo" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "datafusion", "geo-traits", @@ -5369,8 +5369,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arc-swap", "arrow", @@ -5441,8 +5441,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow-array", "arrow-schema", @@ -5465,8 +5465,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow", "arrow-array", @@ -5493,6 +5493,7 @@ dependencies = [ "path_abs", "pin-project", "prost", + "quick-xml 0.41.0", "rand 0.9.5", "reqsign-core", "reqsign-file-read-tokio", @@ -5508,8 +5509,8 @@ dependencies = [ [[package]] name = "lance-jsonb" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "byteorder", "ethnum", @@ -5526,8 +5527,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow-array", "arrow-schema", @@ -5541,8 +5542,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow", "async-trait", @@ -5556,8 +5557,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arc-swap", "arrow", @@ -5611,8 +5612,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow-array", "arrow-buffer", @@ -5626,8 +5627,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow", "arrow-array", @@ -5667,8 +5668,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow-array", "arrow-schema", @@ -5681,8 +5682,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "frostem", "icu_segmenter", @@ -5778,6 +5779,7 @@ dependencies = [ "serial_test", "sha2 0.10.9", "snafu 0.8.9", + "strum_macros 0.28.0", "tempfile", "test-log", "tokenizers", @@ -6818,8 +6820,7 @@ dependencies = [ [[package]] name = "object_store" version = "0.14.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1796bc93603f78c5760a69f2d58badc9618d22adade0a95385bb2adbae4eb94" +source = "git+https://github.com/apache/arrow-rs-object-store.git?rev=c10758c207c0103355f940f0c736fb8ecb157b04#c10758c207c0103355f940f0c736fb8ecb157b04" dependencies = [ "async-trait", "aws-lc-rs", @@ -9445,6 +9446,7 @@ version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ + "indexmap 2.14.0", "itoa", "memchr", "serde", diff --git a/Cargo.toml b/Cargo.toml index 191f77919..6d383267b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,21 +13,21 @@ categories = ["database-implementations"] rust-version = "1.94.0" [workspace.dependencies] -lance = { "version" = "=14.0.0-beta.15", default-features = false, "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=14.0.0-beta.15", default-features = false, "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-jsonb = { version = "=14.0.0-beta.15", tag = "v14.0.0-beta.15", git = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=14.0.0-beta.15", default-features = false, "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { version = "=14.0.0-beta.15", path = "python/vendor/lance-arrow" } +lance = { "version" = "=14.0.0-beta.18", default-features = false, "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=14.0.0-beta.18", default-features = false, "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-jsonb = { version = "=14.0.0-beta.18", tag = "v14.0.0-beta.18", git = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=14.0.0-beta.18", default-features = false, "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { version = "=14.0.0-beta.18", path = "python/vendor/lance-arrow" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow @@ -73,7 +73,7 @@ oauth2 = { version = "5.0", default-features = false } regex = "1.10" semver = "1.0.25" serde = "1" -serde_json = "1" +serde_json = { version = "1", features = ["preserve_order"] } tempfile = "3.5.0" tokio = { version = "1.23", features = ["rt-multi-thread", "sync"] } tonic = { version = "0.14", features = ["tls-native-roots", "tls-aws-lc"] } @@ -84,6 +84,10 @@ chrono = { version = "0.4", default-features = false, features = ["clock"] } # Carry the offset-safe validity fix until it is available in a Lance release. lance-arrow = { path = "python/vendor/lance-arrow" } +[patch.crates-io] +# Lance v14.0.0-beta.18 requires multipart write modes from this upstream revision. +object_store = { git = "https://github.com/apache/arrow-rs-object-store.git", rev = "c10758c207c0103355f940f0c736fb8ecb157b04" } + [profile.ci] debug = "line-tables-only" inherits = "dev" diff --git a/deny.toml b/deny.toml index 20510adc2..d5f42dbff 100644 --- a/deny.toml +++ b/deny.toml @@ -78,6 +78,27 @@ ignore = [ { id = "RUSTSEC-2026-0098", reason = "only affects rustls-webpki 0.101 from legacy aws-smithy/rustls 0.21 chain" }, { id = "RUSTSEC-2026-0099", reason = "only affects rustls-webpki 0.101 from legacy aws-smithy/rustls 0.21 chain" }, + # rustls 0.23.45: four advisories fixed in 0.23.46 (Debug impls leaking + # TLS secrets, ECH rejection buffering, Brotli certificate decompression + # allocation, ServerHello key-exchange downgrade). 0.23.46 was published + # 2026-10-14 and the dependency age policy requires releases to be 48 + # hours old, so the upgrade cannot land yet. Remove these entries and run + # `cargo update -p rustls` once the window has passed. Mirrors lance #9985. + # https://rustsec.org/advisories/RUSTSEC-2026-0346 + # https://rustsec.org/advisories/RUSTSEC-2026-0347 + # https://rustsec.org/advisories/RUSTSEC-2026-0348 + # https://rustsec.org/advisories/RUSTSEC-2026-0349 + { id = "RUSTSEC-2026-0346", reason = "fixed in rustls 0.23.46; blocked on the 48-hour release age policy" }, + { id = "RUSTSEC-2026-0347", reason = "fixed in rustls 0.23.46; blocked on the 48-hour release age policy" }, + { id = "RUSTSEC-2026-0348", reason = "fixed in rustls 0.23.46; blocked on the 48-hour release age policy" }, + { id = "RUSTSEC-2026-0349", reason = "fixed in rustls 0.23.46; blocked on the 48-hour release age policy" }, + + # unsafe-libyaml: unmaintained. Transitive via lance-tokenizer → lindera + # → serde_yaml_ng. No maintained upgrade exists; clearing it requires + # lindera to move off serde_yaml_ng. + # https://rustsec.org/advisories/RUSTSEC-2024-0449 + { id = "RUSTSEC-2024-0449", reason = "transitive via lindera (lance-tokenizer); no maintained upgrade available" }, + # rustls-webpki 0.101.7: reachable panic in CRL parsing. Same legacy # rustls 0.21 chain from aws-smithy-http-client as above. The 0.103 line # we actively use is upgraded to 0.103.13 which contains the fix. @@ -197,4 +218,6 @@ allow-registry = ["https://github.com/rust-lang/crates.io-index"] # releases are cut to crates.io. Allow that specific host. allow-git = [ "https://github.com/lance-format/lance", + # Pinned by Lance for the multipart write mode API (see [patch.crates-io] in Cargo.toml). + "https://github.com/apache/arrow-rs-object-store", ] diff --git a/docs/src/js/classes/Table.md b/docs/src/js/classes/Table.md index 3ac1dd031..59c3209c8 100644 --- a/docs/src/js/classes/Table.md +++ b/docs/src/js/classes/Table.md @@ -143,7 +143,8 @@ containing the new version number of the table after altering the columns. abstract blobColumns(): Promise ``` -Blob v2 columns, including nested dotted paths. +Blob v2 columns addressable by row ID, including struct-nested dotted paths. +Blobs inside lists are omitted because a row ID does not identify an element. #### Returns @@ -527,6 +528,8 @@ table's current checkout. Preserves input order, duplicates, and nulls. Use this for large payloads. See [Table.fetchBlobs](Table.md#fetchblobs) for row-ID validity across versions. +One missing or deleted row ID rejects the entire batch, including on remote +tables; no partial handles are returned. #### Parameters @@ -556,6 +559,10 @@ Bytes for `column` at row IDs from [Query.withRowId](Query.md#withrowid). Reads the table's current checkout. IDs from another version can fail after compaction unless stable row ids are enabled. Results keep input order and duplicates. Null blobs are `null`. Empty blobs are empty buffers. +One missing or deleted row ID rejects the entire batch, including on remote +tables; no partial results are returned. +Blobs inside lists cannot be fetched by row ID. + Remote servers limit each request to 1024 row IDs and 64 MiB of blob bytes. The client splits requests automatically and reads an individual larger blob through the Range route. This method still materializes all bytes in diff --git a/docs/src/js/functions/blob.md b/docs/src/js/functions/blob.md index 20a734cd4..1a65b5133 100644 --- a/docs/src/js/functions/blob.md +++ b/docs/src/js/functions/blob.md @@ -14,6 +14,8 @@ Declares a `lance.blob.v2` column. Query results are descriptors, not payload bytes. Use [Table.fetchBlobs](../classes/Table.md#fetchblobs) or [Table.fetchBlobFiles](../classes/Table.md#fetchblobfiles) to read bytes. +Blobs inside lists can be written and queried as descriptors, but cannot +currently be fetched because a row ID does not identify a list element. ## Parameters diff --git a/docs/src/js/interfaces/ClientConfig.md b/docs/src/js/interfaces/ClientConfig.md index 2b4984b8d..d5ce32117 100644 --- a/docs/src/js/interfaces/ClientConfig.md +++ b/docs/src/js/interfaces/ClientConfig.md @@ -8,6 +8,18 @@ ## Properties +### blobRequestConcurrency? + +```ts +optional blobRequestConcurrency: number; +``` + +Maximum number of concurrent blob HTTP requests across all tables and +blob handles on this connection. Defaults to 8. Can also be set with +`LANCE_CLIENT_BLOB_REQUEST_CONCURRENCY`. Must be greater than zero. + +*** + ### extraHeaders? ```ts diff --git a/docs/src/python/python.md b/docs/src/python/python.md index 3776283dc..24372a83d 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -48,6 +48,169 @@ are ordinary connections. Dropping a database requires it to be empty. ::: lancedb.catalog.DatabaseNames +## Authorization + +Some catalogs support the authorization API. This is an API that can be used to +bind access control lists (ACLs) to individual catalog objects. + +Each ACL contains a subject, an object, and privilege. + +### Subjects + +There are three kinds of subjects: principals, groups, and roles. + +A principal uniquely identifies each user accessing the system. Each LanceDB +Enterprise API key corresponds to a unique principal, and each OpenID user has +a unique principal. + +All principals have an immutable ID which never changes, and is safe to include +in log files. For example, the API key +`sk_AAAQEAYEAUDAOCAJBIFQYDIOB4IBCEQTCQKRMFYYDENBWHA5DYPQ====` maps to the +principal ID `pk_aaaqeayeaudaocajbifqydiob4`. APIs such as `list_acls` will +return the principal ID, never the API key. + +OpenID principals also typically have a human-readable name associated with +them. For example, a principal might be named `bob@example.com` and might have +a principal ID of `8a004400-9443-4fab-9997-b9372a7ca8fd`. Whenever +possible, APIs will return the principal name for convenience. However, +principal names are mutable and can be changed by the identity provider. + +Here is an example of getting information about an API key: +```python +from lancedb.authz import Subject + +catalog = lancedb.connect_catalog("https://catalog.example.com", api_key=api_key) +principal = catalog.authz.get_principal(Subject.principal_api_key(api_key)) +print(f"Found principal {principal.principal_id}") +``` + +OpenID principals can be members of groups. Like principals, groups also have +IDs and human-readable names. It is not possible to modify group membership via +this API; it is managed by the OpenID identity provider. + +Here is an example of getting information about a group based on its name: +```python +from lancedb.authz import Subject + +catalog = lancedb.connect_catalog("https://catalog.example.com", api_key=api_key) +group = catalog.authz.get_group(Subject.group_name("Engineering")) +print(f"Found group {group.group_id} with name {group.group_name}") +``` + +Roles can contain either principals or groups. Unlike groups, they are managed +by the catalog, not the identity provider. Roles can also contain API key +principals. + +Here is an example of managing the `reader` role: +```python +from lancedb.authz import Subject + +catalog = lancedb.connect_catalog("https://catalog.example.com", api_key=api_key) +catalog.authz.add_role_binding(subject=Subject.principal_name("Alice"), role="reader") +``` +It may take a few milliseconds for changes to role bindings to become visible +to all clients. + +More subject types may be added in the future. + +### Objects + +An object is any resource in the system that we can attach authorization rules to. +The Python builders and Rust client share resource validation and wire encoding. +Methods also accept literal strings such as `table:tenant/db:public$events`; +malformed names for known resource types raise `ValueError` locally before a +request is sent. Unknown resource types are preserved for compatibility with +newer servers. + +Object types include: + +The System object, a singleton which represents the entire system. You can +refer to it as `Object.system()`. + +Database objects which represents specific databases. For example, +`Object.database("analytics")` represents the "analytics" database. + +Table objects which represent specific tables. For example, +`Object.table(database="analytics", name="events")` represents the "events" +table inside the "analytics" database. Tables are in the default namespace (which +is named "public") unless a specific namespace is given. + +Similarly, View objects represent specific views. For example, +`Object.view(database="analytics", name="recent_events")` represents the +"recent_events" view inside the "analytics" database. Views are in the default +namespace (which is named "public") unless a specific namespace is given. + +Secret objects identify named credentials, for example, +`Object.secret(database="analytics", name="api-key")`. Function objects identify +registered functions, for example, +`Object.function(database="analytics", name="caption")`. Both default to the +`public` namespace. Pass `namespace_path=["platform", "ml"]` to select a nested +namespace. These objects identify resources for ACL operations; a secret object +contains the credential's name, not its value. + +More object types may be added in the future. + +### Access control lists + +An access control list attaches a specific set of privileges to a specific +(subject, object) pair. So, for example, we might have an access control list +that gives `bob@example.com` the ability to read the foo table, that looks like +this: + +```python +from lancedb.authz import Object, Subject, Privilege + +catalog = lancedb.connect_catalog("https://catalog.example.com", api_key=api_key) +catalog.authz.add_acl( + object=Object.table(database="analytics", name="foo"), + subject=Subject.principal_name("bob@example.com"), + privilege=Privilege.SELECT, +) +``` + +Reading a table generally requires USAGE on its database and namespace, plus +SELECT on the table. Each object has at most one owner. + +OWNERSHIP is a special privilege which grants all the other privileges. By +default, the principal who created an object owns it. Only one subject can own +an object; granting OWNERSHIP to a new subject removes it from the previous +owner. OWNERSHIP cannot be deleted. + +To delegate secret or function creation, grant `CREATE_SECRET` or +`CREATE_FUNCTION` on the containing namespace. This preserves namespace ownership: + +```python +for privilege in [Privilege.CREATE_SECRET, Privilege.CREATE_FUNCTION]: + catalog.authz.add_acl( + object=Object.namespace(database="analytics", namespace_path=["public"]), + subject=Subject.role("developer"), + privilege=privilege, + ) +``` + +Both privileges also accept their string names in synchronous and asynchronous +ACL methods. + +By default, the `list_acls` function lists all the access control entries in +the system. By adding a function parameter, the output is filtered to just ACLs +matching that parameter. For example, this code lists all the ACLs for the +"reader" role: + +```python +from lancedb.authz import Subject + +catalog = lancedb.connect_catalog("https://catalog.example.com", api_key=api_key) +for entry in catalog.authz.list_acls(subject=Subject.role("reader")): + print(entry) +``` + +Or, when using the async API: +```python +catalog = await lancedb.connect_catalog_async(endpoint, api_key=api_key) +async for entry in catalog.authz.list_acls(subject=Subject.role("reader")): + print(entry) +``` + ## Remote SQL Submit SQL against a remote LanceDB database through the connection. @@ -342,6 +505,12 @@ pylance, LanceDB uses a matching `lance.blob.v2` extension type so blob columns still work. Queries return descriptors. Call [`fetch_blob_files`][lancedb.table.Table.fetch_blob_files] for lazy reads or [`fetch_blobs`][lancedb.table.Table.fetch_blobs] for eager bytes. +One missing or deleted row ID rejects the entire blob-read batch; the APIs do +not return partial results. + +Blobs inside lists can be written and queried as descriptors. They are omitted +from `blob_columns()` because row ids do not identify individual list elements, +so the fetch APIs cannot read their bytes yet. ::: lancedb.blob @@ -359,6 +528,8 @@ still work. Queries return descriptors. Call ## Exceptions +::: lancedb.remote.errors.HttpError + ::: lancedb.exceptions.MissingValueError ::: lancedb.exceptions.MissingColumnError diff --git a/docs/web-tests/py/provider_examples.py b/docs/web-tests/py/provider_examples.py new file mode 100644 index 000000000..2ba24fe96 --- /dev/null +++ b/docs/web-tests/py/provider_examples.py @@ -0,0 +1,355 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +"""Reader examples with explicit placeholders. + +These functions are documentation source. test_provider_examples.py runs the +reranker examples offline, with stand-ins for the model and the provider, to +check that each one writes only to its own new directory. Corresponding opt-in +integration tests remain in test_integrations.py and retain real credential +loading. Replace placeholders before running these examples. +""" + + +def embedding_ibm_usage(): + # --8<-- [start:embedding_ibm_usage] + import tempfile + from pathlib import Path + + import lancedb + from lancedb.embeddings import EmbeddingFunctionRegistry + from lancedb.pydantic import LanceModel, Vector + + registry = EmbeddingFunctionRegistry.get_instance() + registry.set_var("watsonx-api-key", "your-api-key") + watsonx_embed = registry.get("watsonx").create( + name="ibm/slate-125m-english-rtrvr", + api_key="$var:watsonx-api-key", + project_id="your-project-id", + ) + + class TextModel(LanceModel): + text: str = watsonx_embed.SourceField() + vector: Vector(watsonx_embed.ndims()) = watsonx_embed.VectorField() + + data = [ + {"text": "hello world"}, + {"text": "goodbye world"}, + ] + + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "watsonx-demo")) + tbl = db.create_table("watsonx_test", schema=TextModel, mode="overwrite") + tbl.add(data) + + rs = tbl.search("hello").limit(1).to_pandas() + print(rs.head()) + # --8<-- [end:embedding_ibm_usage] + + +def embedding_jina_text(): + # --8<-- [start:embedding_jina_text] + import os + import tempfile + from pathlib import Path + + import lancedb + from lancedb.embeddings import EmbeddingFunctionRegistry + from lancedb.pydantic import LanceModel, Vector + + os.environ["JINA_API_KEY"] = "your-api-key" + + jina_embed = ( + EmbeddingFunctionRegistry.get_instance() + .get("jina") + .create(name="jina-embeddings-v2-base-en") + ) + + class TextModel(LanceModel): + text: str = jina_embed.SourceField() + vector: Vector(jina_embed.ndims()) = jina_embed.VectorField() + + data = [{"text": "hello world"}, {"text": "goodbye world"}] + + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "jina-text")) + tbl = db.create_table("test", schema=TextModel, mode="overwrite") + + tbl.add(data) + # --8<-- [end:embedding_jina_text] + + +def embedding_jina_multimodal(): + # --8<-- [start:embedding_jina_multimodal] + import os + import tempfile + from pathlib import Path + + import lancedb + import pandas as pd + import requests + from lancedb.embeddings import get_registry + from lancedb.pydantic import LanceModel, Vector + + os.environ["JINA_API_KEY"] = "your-api-key" + + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "jina-images")) + func = get_registry().get("jina").create() + + class Images(LanceModel): + label: str + image_uri: str = func.SourceField() + image_bytes: bytes = func.SourceField() + vector: Vector(func.ndims()) = func.VectorField() + vec_from_bytes: Vector(func.ndims()) = func.VectorField() + + table = db.create_table("images", schema=Images) + labels = ["cat", "cat", "dog", "dog", "horse", "horse"] + uris = [ + "http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg", + "http://farm1.staticflickr.com/134/332220238_da527d8140_z.jpg", + "http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg", + "http://farm5.staticflickr.com/4092/5017326486_1f46057f5f_z.jpg", + "http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg", + "http://farm6.staticflickr.com/5142/5835678453_4f3a4edb45_z.jpg", + ] + image_bytes = [requests.get(uri).content for uri in uris] + table.add( + pd.DataFrame({"label": labels, "image_uri": uris, "image_bytes": image_bytes}) + ) + # --8<-- [end:embedding_jina_multimodal] + + +def reranking_cohere_usage(): + # --8<-- [start:reranking_cohere_usage] + import tempfile + from pathlib import Path + + import lancedb + from lancedb.embeddings import get_registry + from lancedb.pydantic import LanceModel, Vector + from lancedb.rerankers import CohereReranker + + embedder = get_registry().get("sentence-transformers").create() + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "cohere-rerank")) + + class Schema(LanceModel): + text: str = embedder.SourceField() + vector: Vector(embedder.ndims()) = embedder.VectorField() + + data = [ + {"text": "hello world"}, + {"text": "goodbye world"}, + ] + tbl = db.create_table("test", schema=Schema) + tbl.add(data) + reranker = CohereReranker(api_key="your-api-key") + + # Run vector search with a reranker + vector_results = tbl.search("hello").rerank(reranker=reranker).to_list() + print(vector_results) + + # Build the full-text index before either FTS or hybrid search. + tbl.create_fts_index("text", replace=True) + + # Run FTS search with a reranker + fts_results = ( + tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list() + ) + print(fts_results) + + # Run hybrid search with a reranker + hybrid_results = ( + tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() + ) + print(hybrid_results) + # --8<-- [end:reranking_cohere_usage] + + +def reranking_jina_usage(): + # --8<-- [start:reranking_jina_usage] + import os + import tempfile + from pathlib import Path + + import lancedb + from lancedb.embeddings import get_registry + from lancedb.pydantic import LanceModel, Vector + from lancedb.rerankers import JinaReranker + + os.environ["JINA_API_KEY"] = "your-api-key" + embedder = get_registry().get("jina").create() + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "jina-rerank")) + + class Schema(LanceModel): + text: str = embedder.SourceField() + vector: Vector(embedder.ndims()) = embedder.VectorField() + + data = [ + {"text": "hello world"}, + {"text": "goodbye world"}, + ] + tbl = db.create_table("test", schema=Schema) + tbl.add(data) + reranker = JinaReranker(api_key="your-api-key") + + # Run vector search with a reranker + vector_results = tbl.search("hello").rerank(reranker=reranker).to_list() + print(vector_results) + + # Build the full-text index before either FTS or hybrid search. + tbl.create_fts_index("text", replace=True) + + # Run FTS search with a reranker + fts_results = ( + tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list() + ) + print(fts_results) + + # Run hybrid search with a reranker + hybrid_results = ( + tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() + ) + print(hybrid_results) + # --8<-- [end:reranking_jina_usage] + + +def reranking_watsonx_usage(): + # --8<-- [start:reranking_watsonx_usage] + import tempfile + from pathlib import Path + + import lancedb + from lancedb.embeddings import get_registry + from lancedb.pydantic import LanceModel, Vector + from lancedb.rerankers import WatsonxReranker + + embedder = get_registry().get("sentence-transformers").create() + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "watsonx-rerank")) + + class Schema(LanceModel): + text: str = embedder.SourceField() + vector: Vector(embedder.ndims()) = embedder.VectorField() + + data = [ + {"text": "hello world"}, + {"text": "goodbye world"}, + ] + tbl = db.create_table("test", schema=Schema) + tbl.add(data) + + # Supply your API key and project ID (or use space_id instead of project_id). + reranker = WatsonxReranker( + api_key="your-api-key", + project_id="your-project-id", + ) + + # Run vector search with a reranker + vector_results = tbl.search("hello").rerank(reranker=reranker).to_list() + print(vector_results) + + # Build the full-text index before either FTS or hybrid search. + tbl.create_fts_index("text", replace=True) + + # Run FTS search with a reranker + fts_results = ( + tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list() + ) + print(fts_results) + + # Run hybrid search with a reranker + hybrid_results = ( + tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() + ) + print(hybrid_results) + # --8<-- [end:reranking_watsonx_usage] + + +def reranking_openai_usage(): + # --8<-- [start:reranking_openai_usage] + import tempfile + from pathlib import Path + + import lancedb + from lancedb.embeddings import get_registry + from lancedb.pydantic import LanceModel, Vector + from lancedb.rerankers import OpenaiReranker + + embedder = get_registry().get("sentence-transformers").create() + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "openai-rerank")) + + class Schema(LanceModel): + text: str = embedder.SourceField() + vector: Vector(embedder.ndims()) = embedder.VectorField() + + data = [ + {"text": "hello world"}, + {"text": "goodbye world"}, + ] + tbl = db.create_table("test", schema=Schema) + tbl.add(data) + reranker = OpenaiReranker(api_key="your-api-key") + + # Run vector search with a reranker + vector_results = tbl.search("hello").rerank(reranker=reranker).to_list() + print(vector_results) + + # Build the full-text index before either FTS or hybrid search. + tbl.create_fts_index("text", replace=True) + + # Run FTS search with a reranker + fts_results = ( + tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list() + ) + print(fts_results) + + # Run hybrid search with a reranker + hybrid_results = ( + tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() + ) + print(hybrid_results) + # --8<-- [end:reranking_openai_usage] + + +def reranking_voyageai_usage(): + # --8<-- [start:reranking_voyageai_usage] + import tempfile + from pathlib import Path + + import lancedb + from lancedb.embeddings import get_registry + from lancedb.pydantic import LanceModel, Vector + from lancedb.rerankers import VoyageAIReranker + + embedder = get_registry().get("sentence-transformers").create() + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "voyageai-rerank")) + + class Schema(LanceModel): + text: str = embedder.SourceField() + vector: Vector(embedder.ndims()) = embedder.VectorField() + + data = [ + {"text": "hello world"}, + {"text": "goodbye world"}, + ] + tbl = db.create_table("test", schema=Schema) + tbl.add(data) + reranker = VoyageAIReranker(api_key="your-api-key", model_name="rerank-2") + + # Run vector search with a reranker + vector_results = tbl.search("hello").rerank(reranker=reranker).to_list() + print(vector_results) + + # Build the full-text index before either FTS or hybrid search. + tbl.create_fts_index("text", replace=True) + + # Run FTS search with a reranker + fts_results = ( + tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list() + ) + print(fts_results) + + # Run hybrid search with a reranker + hybrid_results = ( + tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() + ) + print(hybrid_results) + # --8<-- [end:reranking_voyageai_usage] diff --git a/docs/web-tests/py/test_connection.py b/docs/web-tests/py/test_connection.py index 783ac2d8d..f5c1f2b22 100644 --- a/docs/web-tests/py/test_connection.py +++ b/docs/web-tests/py/test_connection.py @@ -4,6 +4,8 @@ from pathlib import Path import shutil +import pytest + def test_connection(): # --8<-- [start:connect] @@ -29,19 +31,16 @@ async def connect_async_example(): def connect_enterprise_quickstart_config(): + # --8<-- [start:connect_enterprise_quickstart] + import os + import lancedb - # --8<-- [start:connect_enterprise_quickstart] - uri = "db://your-database-uri" - api_key = "your-api-key" - region = "us-east-1" - host_override = "https://your-enterprise-endpoint.com" - db = lancedb.connect( - uri=uri, - api_key=api_key, - region=region, - host_override=host_override, + uri=os.environ["LANCEDB_URI"], # db://your-database + api_key=os.environ["LANCEDB_API_KEY"], + region=os.environ.get("LANCEDB_REGION", "us-east-1"), + host_override=os.environ["LANCEDB_HOST_OVERRIDE"], # https://your-endpoint ) # --8<-- [end:connect_enterprise_quickstart] return db @@ -57,17 +56,29 @@ def test_connect_enterprise_quickstart(monkeypatch): return object() monkeypatch.setattr(lancedb, "connect", fake_connect) + monkeypatch.setenv("LANCEDB_URI", "db://your-database") + monkeypatch.setenv("LANCEDB_API_KEY", "your-api-key") + monkeypatch.delenv("LANCEDB_REGION", raising=False) + monkeypatch.setenv("LANCEDB_HOST_OVERRIDE", "https://your-endpoint") - db = connect_enterprise_quickstart_config() - assert db is not None + assert connect_enterprise_quickstart_config() is not None assert captured == { - "uri": "db://your-database-uri", + "uri": "db://your-database", "api_key": "your-api-key", "region": "us-east-1", - "host_override": "https://your-enterprise-endpoint.com", + "host_override": "https://your-endpoint", } +def test_connect_enterprise_quickstart_requires_an_endpoint(monkeypatch): + monkeypatch.setenv("LANCEDB_URI", "db://your-database") + monkeypatch.setenv("LANCEDB_API_KEY", "your-api-key") + monkeypatch.delenv("LANCEDB_HOST_OVERRIDE", raising=False) + + with pytest.raises(KeyError, match="LANCEDB_HOST_OVERRIDE"): + connect_enterprise_quickstart_config() + + def connect_object_storage_config(): # --8<-- [start:connect_object_storage] import lancedb @@ -105,14 +116,26 @@ def namespace_table_ops_example(): mode="create", # use "overwrite" only if you want to replace existing table ) - # Verify - print(db.list_namespaces()) # ['prod'] - print(db.list_namespaces(namespace_path=["prod"])) # ['recommendations', 'search'] - print(db.list_tables(namespace_path=["prod", "search"])) # ['user'] - print(db.list_tables(namespace_path=["prod", "recommendations"])) # ['user'] + # Verify: listings return a response with the names and a page token + print(db.list_namespaces().namespaces) # ['prod'] + print(db.list_namespaces(namespace_path=["prod"]).namespaces) + # ['recommendations', 'search'] + print(db.list_tables(namespace_path=["prod", "search"]).tables) # ['user'] + print(db.list_tables(namespace_path=["prod", "recommendations"]).tables) # ['user'] # --8<-- [end:namespace_table_ops] +def test_namespace_table_ops(tmp_path, monkeypatch, capsys): + monkeypatch.chdir(tmp_path) + namespace_table_ops_example() + assert capsys.readouterr().out.splitlines() == [ + "['prod']", + "['recommendations', 'search']", + "['user']", + "['user']", + ] + + def namespace_admin_ops_example(): # --8<-- [start:namespace_admin_ops] import lancedb @@ -124,8 +147,8 @@ def namespace_admin_ops_example(): db.create_namespace(["prod", "search"]) child_namespaces = db.list_namespaces(namespace_path=["prod"]).namespaces - print(f"Child namespaces under {namespace}: {child_namespaces}") - # Child namespaces under ['prod', 'search']: ['search'] + print(f"Child namespaces under ['prod']: {child_namespaces}") + # Child namespaces under ['prod']: ['search'] metadata = db.describe_namespace(["prod", "search"]) print(f"Metadata for namespace {namespace}: {metadata}") @@ -134,7 +157,17 @@ def namespace_admin_ops_example(): db.drop_namespace(["prod", "search"], mode="skip") db.drop_namespace(["prod"], mode="skip") # --8<-- [end:namespace_admin_ops] - return child_namespaces, metadata + return db, child_namespaces, metadata + + +def test_namespace_admin_ops(tmp_path, monkeypatch, capsys): + monkeypatch.chdir(tmp_path) + db, child_namespaces, metadata = namespace_admin_ops_example() + assert child_namespaces == ["search"] + assert capsys.readouterr().out.splitlines()[0] == ( + "Child namespaces under ['prod']: ['search']" + ) + assert db.list_namespaces().namespaces == [] async def connect_object_storage_config_async(): # --8<-- [start:connect_object_storage_async] @@ -146,3 +179,32 @@ async def connect_object_storage_config_async(): # --8<-- [end:connect_object_storage_async] return async_db + + +def test_connect_enterprise_placeholders(monkeypatch): + import lancedb + + captured = {} + + def fake_connect(**kwargs): + captured.update(kwargs) + return object() + + monkeypatch.setattr(lancedb, "connect", fake_connect) + # --8<-- [start:connect_enterprise_placeholders] + import lancedb + + db = lancedb.connect( + uri="db://your-database", + api_key="your-api-key", + region="us-east-1", + host_override="https://your-endpoint", + ) + # --8<-- [end:connect_enterprise_placeholders] + assert db is not None + assert captured == { + "uri": "db://your-database", + "api_key": "your-api-key", + "region": "us-east-1", + "host_override": "https://your-endpoint", + } diff --git a/docs/web-tests/py/test_curation_sampling.py b/docs/web-tests/py/test_curation_sampling.py new file mode 100644 index 000000000..d50b8a43f --- /dev/null +++ b/docs/web-tests/py/test_curation_sampling.py @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + + +def test_sample_dataset(): + # --8<-- [start:sample_dataset] + from tempfile import TemporaryDirectory + + import lancedb + from lancedb.permutation import Permutation, permutation_builder + + with TemporaryDirectory(prefix="lancedb-sampling-") as uri: + db = lancedb.connect(uri) + table = db.create_table("records", [{"id": i} for i in range(10)]) + splits = ( + permutation_builder(table).split_random(counts=[3, 7], seed=42).execute() + ) + sample = Permutation.from_tables(table, splits, split=0).with_format("arrow") + selected_ids = [] + for batch in sample.iter(batch_size=3): + selected_ids.extend(batch.column("id").to_pylist()) + print(f"Selected {len(selected_ids)} rows: {selected_ids}") + # --8<-- [end:sample_dataset] + + assert len(selected_ids) == 3 + assert len(set(selected_ids)) == 3 + assert all(0 <= value < 10 for value in selected_ids) diff --git a/docs/web-tests/py/test_function.py b/docs/web-tests/py/test_function.py index 629ab89bc..f66a50aec 100644 --- a/docs/web-tests/py/test_function.py +++ b/docs/web-tests/py/test_function.py @@ -4,16 +4,18 @@ """Examples for docs/web/management/function.mdx. Registering a Function and filling its column need a LanceDB Enterprise -deployment, so `test_function_enterprise` runs only when the connection -variables the Enterprise examples already use are set: LANCEDB_URI, -LANCEDB_API_KEY and LANCEDB_HOST_OVERRIDE, plus LANCEDB_REGION if the -deployment needs one. The walkthrough calls its table `function_demo` and its -Function `double`; the test creates both under names unique to its run, never -replaces an existing object, and drops only what it created. +deployment, so `test_function_enterprise` runs only when +LANCEDB_DOCS_TEST_DEPLOYMENT=1 asks for one, with the connection variables the +Enterprise examples already use: LANCEDB_URI, LANCEDB_API_KEY and +LANCEDB_HOST_OVERRIDE, plus LANCEDB_REGION if the deployment needs one. The +walkthrough calls its table `function_demo` and its Function `double`; the test +creates both under names unique to its run, never replaces an existing object, +and drops only what it created. The other tests run locally and check the client side of the same code: what -`@udf` sends for registration, how a registered version binds a column, and -where a local connection stops. They do not run a Function remotely. +`@udf` sends for registration, how a registered version binds a column, where a +local connection stops, and when the Enterprise test connects. They do not run +a Function remotely. """ import json @@ -70,12 +72,17 @@ def registered_version(definition) -> FunctionVersion: return FunctionVersion.from_json(json.dumps(reply)) -def require_env(var_name: str) -> str: - """Skip the test unless the required environment variable is provided.""" - value = os.environ.get(var_name) - if not value: - pytest.skip(f"Set {var_name} to run this Enterprise example") - return value +DEPLOYMENT = ("LANCEDB_URI", "LANCEDB_HOST_OVERRIDE", "LANCEDB_API_KEY") +# The connection settings alone never send this test to a deployment. +OPT_IN = "LANCEDB_DOCS_TEST_DEPLOYMENT" + + +def require_deployment(): + if os.environ.get(OPT_IN) != "1": + pytest.skip(f"set {OPT_IN}=1 and {', '.join(DEPLOYMENT)} to use a deployment") + missing = [name for name in DEPLOYMENT if not os.environ.get(name)] + if missing: + pytest.fail(f"{OPT_IN}=1 needs {', '.join(missing)}") def test_function_runs_locally_as_python(): @@ -160,9 +167,7 @@ def test_function_version_binds_a_column(tmp_path): def test_function_enterprise(monkeypatch): - require_env("LANCEDB_URI") - require_env("LANCEDB_API_KEY") - require_env("LANCEDB_HOST_OVERRIDE") + require_deployment() # --8<-- [start:function_connect] import os @@ -240,3 +245,91 @@ def test_function_enterprise(monkeypatch): db.drop_table(table_name) if version is not None: db.drop_function(version.name, version=version.version) + + +class ReachedConnection(Exception): + pass + + +SETTINGS = { + "LANCEDB_URI": "db://docs-test", + "LANCEDB_HOST_OVERRIDE": "https://deployment.invalid", + "LANCEDB_API_KEY": "docs-test-key", +} + + +@pytest.fixture +def connect_attempts(monkeypatch): + """`lancedb.connect`, replaced so that it records each call and stops.""" + attempts = [] + + def connect(**kwargs): + attempts.append(kwargs) + raise ReachedConnection + + monkeypatch.setattr(lancedb, "connect", connect) + for name in (OPT_IN, *DEPLOYMENT, "LANCEDB_REGION"): + monkeypatch.delenv(name, raising=False) + return attempts + + +def enterprise_outcome(monkeypatch): + try: + test_function_enterprise(monkeypatch) + except (pytest.skip.Exception, pytest.fail.Exception, ReachedConnection) as outcome: + return outcome + + +@pytest.mark.parametrize( + "opt_in", + [None, "", "0", "true", "TRUE", "yes", "on", " 1", "1 ", "01", "1.0", "11"], +) +def test_function_enterprise_needs_the_exact_opt_in( + monkeypatch, connect_attempts, opt_in +): + for name, value in SETTINGS.items(): + monkeypatch.setenv(name, value) + if opt_in is not None: + monkeypatch.setenv(OPT_IN, opt_in) + outcome = enterprise_outcome(monkeypatch) + assert isinstance(outcome, pytest.skip.Exception), outcome + assert str(outcome).startswith(f"set {OPT_IN}=1 ") + assert connect_attempts == [] + + +@pytest.mark.parametrize("blank", ["unset", "empty"]) +@pytest.mark.parametrize( + "missing", [*([name] for name in DEPLOYMENT), list(DEPLOYMENT)] +) +def test_function_enterprise_fails_when_asked_without_settings( + monkeypatch, connect_attempts, missing, blank +): + monkeypatch.setenv(OPT_IN, "1") + for name, value in SETTINGS.items(): + if name not in missing: + monkeypatch.setenv(name, value) + elif blank == "empty": + monkeypatch.setenv(name, "") + outcome = enterprise_outcome(monkeypatch) + assert isinstance(outcome, pytest.fail.Exception), outcome + assert str(outcome) == f"{OPT_IN}=1 needs {', '.join(missing)}" + assert connect_attempts == [] + + +@pytest.mark.parametrize("region", [None, "eu-west-1"]) +def test_function_enterprise_connects_when_asked(monkeypatch, connect_attempts, region): + monkeypatch.setenv(OPT_IN, "1") + for name, value in SETTINGS.items(): + monkeypatch.setenv(name, value) + if region: + monkeypatch.setenv("LANCEDB_REGION", region) + outcome = enterprise_outcome(monkeypatch) + assert isinstance(outcome, ReachedConnection), outcome + assert connect_attempts == [ + { + "uri": "db://docs-test", + "api_key": "docs-test-key", + "region": region or "us-east-1", + "host_override": "https://deployment.invalid", + } + ] diff --git a/docs/web-tests/py/test_ingestion_examples.py b/docs/web-tests/py/test_ingestion_examples.py new file mode 100644 index 000000000..0dedc1458 --- /dev/null +++ b/docs/web-tests/py/test_ingestion_examples.py @@ -0,0 +1,739 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + + +def test_create_table_from_dicts(): + # --8<-- [start:create_table_from_dicts] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + data = [ + {"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7}, + {"vector": [0.2, 1.8], "lat": 40.1, "long": -74.1}, + ] + db.create_table("test_table", data, mode="overwrite") + tbl = db["test_table"] + tbl.head() + + print("Rows:", tbl.count_rows()) + # --8<-- [end:create_table_from_dicts] + assert tbl.count_rows() == 2 + + +def test_create_table_conflict_handling(): + # --8<-- [start:create_table_conflict_handling] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7}] + db.create_table("conflict_table", data) + # Idempotent open: reuse the existing table if it exists. + # The provided data is ignored; the schema is validated against the + # existing table and a mismatch raises an error. + tbl = db.create_table("conflict_table", data, exist_ok=True) + + # Overwrite: replace the contents with the provided data. + tbl = db.create_table("conflict_table", data, mode="overwrite") + print("Rows:", tbl.count_rows()) + # --8<-- [end:create_table_conflict_handling] + + +def test_create_table_custom_schema(): + # --8<-- [start:create_table_custom_schema] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import pyarrow as pa + + custom_schema = pa.schema( + [ + pa.field("vector", pa.list_(pa.float32(), 4)), + pa.field("lat", pa.float32()), + pa.field("long", pa.float32()), + ] + ) + + data = [ + {"vector": [1.1, 1.2, 1.3, 1.4], "lat": 45.5, "long": -122.7}, + {"vector": [0.2, 1.8, 0.4, 3.6], "lat": 40.1, "long": -74.1}, + ] + tbl = db.create_table( + "my_table_custom_schema", data, schema=custom_schema, mode="overwrite" + ) + + print("Rows:", tbl.count_rows()) + # --8<-- [end:create_table_custom_schema] + assert tbl.count_rows() == 2 + + +def test_create_table_from_pandas(): + # --8<-- [start:create_table_from_pandas] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import pandas as pd + + data = pd.DataFrame( + { + "vector": [[1.1, 1.2, 1.3, 1.4], [0.2, 1.8, 0.4, 3.6]], + "lat": [45.5, 40.1], + "long": [-122.7, -74.1], + } + ) + db.create_table("my_table_pandas", data, mode="overwrite") + db["my_table_pandas"].head() + # --8<-- [end:create_table_from_pandas] + + +def test_create_table_from_polars(): + # --8<-- [start:create_table_from_polars] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import polars as pl + + data = pl.DataFrame( + { + "vector": [[3.1, 4.1], [5.9, 26.5]], + "item": ["foo", "bar"], + "price": [10.0, 20.0], + } + ) + tbl = db.create_table("my_table_pl", data, mode="overwrite") + + print("Rows:", tbl.count_rows()) + # --8<-- [end:create_table_from_polars] + assert tbl.count_rows() == 2 + + +def test_create_table_from_arrow(): + # --8<-- [start:create_table_from_arrow] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import numpy as np + import pyarrow as pa + + dim = 16 + total = 2 + schema = pa.schema( + [ + pa.field("vector", pa.list_(pa.float16(), dim)), + pa.field("text", pa.string()), + ] + ) + data = pa.Table.from_arrays( + [ + pa.array( + [np.random.randn(dim).astype(np.float16) for _ in range(total)], + pa.list_(pa.float16(), dim), + ), + pa.array(["foo", "bar"]), + ], + ["vector", "text"], + ) + tbl = db.create_table("f16_tbl", data, schema=schema, mode="overwrite") + + print("Rows:", tbl.count_rows()) + # --8<-- [end:create_table_from_arrow] + assert tbl.count_rows() == 2 + + +def test_create_table_from_pydantic(): + # --8<-- [start:create_table_from_pydantic] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + from lancedb.pydantic import LanceModel, Vector + + class Content(LanceModel): + movie_id: int + vector: Vector(128) + genres: str + title: str + imdb_id: int + + @property + def imdb_url(self) -> str: + return f"https://www.imdb.com/title/tt{self.imdb_id}" + + tbl = db.create_table("movielens_small", schema=Content, mode="overwrite") + + print("Rows:", tbl.count_rows()) + # --8<-- [end:create_table_from_pydantic] + assert tbl.count_rows() == 0 + + +def test_create_table_nested_schema(): + # --8<-- [start:create_table_nested_schema] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + from lancedb.pydantic import LanceModel, Vector + from pydantic import BaseModel + + class Document(BaseModel): + content: str + source: str + + class NestedSchema(LanceModel): + id: str + vector: Vector(1536) + document: Document + + tbl = db.create_table("nested_table", schema=NestedSchema, mode="overwrite") + + print("Rows:", tbl.count_rows()) + # --8<-- [end:create_table_nested_schema] + assert tbl.count_rows() == 0 + + +def test_create_empty_table(): + # --8<-- [start:create_empty_table] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import pyarrow as pa + + schema = pa.schema( + [ + pa.field("vector", pa.list_(pa.float32(), 2)), + pa.field("item", pa.string()), + pa.field("price", pa.float32()), + ] + ) + tbl = db.create_table("test_empty_table", schema=schema, mode="overwrite") + + print("Rows:", tbl.count_rows()) + # --8<-- [end:create_empty_table] + assert tbl.count_rows() == 0 + + +def test_create_empty_table_pydantic(): + # --8<-- [start:create_empty_table_pydantic] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + from lancedb.pydantic import LanceModel, Vector + + class Item(LanceModel): + vector: Vector(2) + item: str + price: float + + tbl = db.create_table( + "test_empty_table_new", schema=Item.to_arrow_schema(), mode="overwrite" + ) + + print("Rows:", tbl.count_rows()) + # --8<-- [end:create_empty_table_pydantic] + assert tbl.count_rows() == 0 + + +def test_open_existing_table(): + # --8<-- [start:open_existing_table] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + # Create a table first + data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7}] + db.create_table("test_table", data, mode="overwrite") + + # List table names + print(db.list_tables().tables) + + # Open existing table + tbl = db.open_table("test_table") + + print("Rows:", tbl.count_rows()) + # --8<-- [end:open_existing_table] + assert tbl.count_rows() == 1 + + +def test_drop_table(): + # --8<-- [start:drop_table] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + # Create a table first + data = [{"vector": [1.1, 1.2], "lat": 45.5}] + db.create_table("my_table", data, mode="overwrite") + + # Drop the table + db.drop_table("my_table") + # --8<-- [end:drop_table] + + +def test_add_from_dataset(): + # --8<-- [start:add_from_dataset] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + from pathlib import Path + + import pyarrow as pa + import pyarrow.parquet as pq + + data_path = str(Path(uri) / "input.parquet") + pq.write_table(pa.table({"id": [1, 2], "name": ["Alice", "Bob"]}), data_path) + import pyarrow.dataset as ds + + dataset = ds.dataset(data_path, format="parquet") + table = db.create_table("my_table", schema=dataset.schema, mode="overwrite") + table.add(dataset) + + print("Rows:", table.count_rows()) + # --8<-- [end:add_from_dataset] + assert table.count_rows() == 2 + + +def test_create_table_from_iterator(): + # --8<-- [start:create_table_from_iterator] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import pyarrow as pa + + schema = pa.schema( + [ + pa.field("vector", pa.list_(pa.float32(), 4)), + pa.field("item", pa.utf8()), + pa.field("price", pa.float32()), + ] + ) + + def make_batches(): + for i in range(5): + yield pa.RecordBatch.from_arrays( + [ + pa.array( + [[3.1, 4.1, 5.1, 6.1], [5.9, 26.5, 4.7, 32.8]], + pa.list_(pa.float32(), 4), + ), + pa.array(["foo", "bar"]), + pa.array([10.0, 20.0]), + ], + ["vector", "item", "price"], + ) + + table = db.create_table("batched_table", schema=schema) + table.add(make_batches()) + + print("Rows:", table.count_rows()) + # --8<-- [end:create_table_from_iterator] + assert table.count_rows() == 10 + + +def test_update_operation(): + # --8<-- [start:update_operation] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import pyarrow as pa + + table = db.create_table( + "users_example", + data=pa.table( + { + "id": [1, 2], + "name": ["Alice", "Bob"], + "login_count": [10, 20], + } + ), + mode="overwrite", + ) + table.update(where="id = 2", values={"name": "Bobby"}) + + print("Rows:", table.count_rows()) + # --8<-- [end:update_operation] + assert table.count_rows() == 2 + assert table.search().where("id = 2").to_list()[0]["name"] == "Bobby" + + +def test_update_using_sql(): + # --8<-- [start:update_using_sql] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import pyarrow as pa + + table = db.create_table( + "users_example", + data=pa.table( + { + "id": [1, 2], + "name": ["Alice", "Bob"], + "login_count": [10, 20], + } + ), + mode="overwrite", + ) + table.update(where="id = 2", values_sql={"login_count": "login_count + 1"}) + + print("Rows:", table.count_rows()) + # --8<-- [end:update_using_sql] + assert table.count_rows() == 2 + assert table.search().where("id = 2").to_list()[0]["login_count"] == 21 + + +def test_merge_matched_update_only(): + # --8<-- [start:merge_matched_update_only] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import pyarrow as pa + + table = db.create_table( + "users_example", + data=pa.table( + { + "id": [1, 2], + "name": ["Alice", "Bob"], + "login_count": [10, 20], + } + ), + mode="overwrite", + ) + + incoming_users = pa.table( + { + "id": [2, 3], + "name": ["Bobby", "Charlie"], + "login_count": [21, 5], + } + ) + + (table.merge_insert("id").when_matched_update_all().execute(incoming_users)) + + print("Rows:", table.count_rows()) + # --8<-- [end:merge_matched_update_only] + assert table.count_rows() == 2 + + +def test_insert_if_not_exists(): + # --8<-- [start:insert_if_not_exists] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import pyarrow as pa + + table = db.create_table( + "users_example", + data=pa.table( + { + "id": [1, 2], + "name": ["Alice", "Bob"], + "login_count": [10, 20], + } + ), + mode="overwrite", + ) + + incoming_users = pa.table( + { + "id": [2, 3], + "name": ["Bobby", "Charlie"], + "login_count": [21, 5], + } + ) + + (table.merge_insert("id").when_not_matched_insert_all().execute(incoming_users)) + + print("Rows:", table.count_rows()) + # --8<-- [end:insert_if_not_exists] + assert table.count_rows() == 3 + + +def test_merge_update_insert(): + # --8<-- [start:merge_update_insert] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import pyarrow as pa + + table = db.create_table( + "users_example", + data=pa.table( + { + "id": [1, 2], + "name": ["Alice", "Bob"], + "login_count": [10, 20], + } + ), + mode="overwrite", + ) + + incoming_users = pa.table( + { + "id": [2, 3], + "name": ["Bobby", "Charlie"], + "login_count": [21, 5], + } + ) + + ( + table.merge_insert("id") + .when_matched_update_all() + .when_not_matched_insert_all() + .execute(incoming_users) + ) + + print("Rows:", table.count_rows()) + # --8<-- [end:merge_update_insert] + assert table.count_rows() == 3 + assert table.search().where("id = 2").to_list()[0]["login_count"] == 21 + + +def test_merge_delete_missing_by_source(): + # --8<-- [start:merge_delete_missing_by_source] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import pyarrow as pa + + table = db.create_table( + "users_example", + data=pa.table( + { + "id": [1, 2, 3], + "name": ["Alice", "Bob", "Charlie"], + "login_count": [10, 20, 5], + } + ), + mode="overwrite", + ) + + incoming_users = pa.table( + { + "id": [2, 3], + "name": ["Bobby", "Charlie"], + "login_count": [21, 5], + } + ) + + ( + table.merge_insert("id") + .when_matched_update_all() + .when_not_matched_insert_all() + .when_not_matched_by_source_delete() + .execute(incoming_users) + ) + + print("Rows:", table.count_rows()) + # --8<-- [end:merge_delete_missing_by_source] + assert table.count_rows() == 2 + assert table.count_rows("id = 1") == 0 + + +def test_merge_partial_columns(): + # --8<-- [start:merge_partial_columns] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + import pyarrow as pa + + table = db.create_table( + "users_example", + data=pa.table( + { + "id": [1, 2], + "name": ["Alice", "Bob"], + "login_count": [10, 20], + } + ), + mode="overwrite", + ) + + incoming_users = pa.table( + { + "id": [2, 3], + "name": ["Bobby", "Charlie"], + } + ) + + ( + table.merge_insert("id") + .when_matched_update_all() + .when_not_matched_insert_all() + .execute(incoming_users) + ) + + print("Rows:", table.count_rows()) + # --8<-- [end:merge_partial_columns] + assert table.count_rows() == 3 + assert table.search().where("id = 3").to_list()[0]["login_count"] is None + + +def test_write_workflow(): + # --8<-- [start:write_workflow] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + table = db.create_table( + "users", + [ + {"id": 1, "name": "Alice", "login_count": 10}, + {"id": 2, "name": "Bob", "login_count": 20}, + ], + ) + table.add([{"id": 3, "name": "Charlie", "login_count": 5}]) + table.update(where="id = 2", values={"name": "Bobby"}) + table.update(where="id = 2", values_sql={"login_count": "login_count + 1"}) + ( + table.merge_insert("id") + .when_matched_update_all() + .when_not_matched_insert_all() + .execute( + [ + {"id": 3, "name": "Charlie", "login_count": 6}, + {"id": 4, "name": "Dana", "login_count": 1}, + ] + ) + ) + print("Rows selected for deletion:", table.search().where("id = 4").to_list()) + table.delete("id = 4") + rows = sorted(table.search().to_list(), key=lambda row: row["id"]) + print(rows) + # --8<-- [end:write_workflow] + + assert rows == [ + {"id": 1, "name": "Alice", "login_count": 10}, + {"id": 2, "name": "Bobby", "login_count": 21}, + {"id": 3, "name": "Charlie", "login_count": 6}, + ] + + +def test_append_rows(): + # --8<-- [start:append_rows] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + table = db.create_table("users", [{"id": 1, "name": "Alice"}]) + table.add([{"id": 2, "name": "Bob"}, {"id": 3, "name": "Charlie"}]) + rows = sorted(table.search().to_list(), key=lambda row: row["id"]) + print(rows) + # --8<-- [end:append_rows] + + assert [row["id"] for row in rows] == [1, 2, 3] + + +def test_delete_rows(): + # --8<-- [start:delete_rows] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + table = db.create_table( + "users", + [ + {"id": 1, "name": "Alice"}, + {"id": 2, "name": "Bob"}, + {"id": 3, "name": "Charlie"}, + ], + ) + predicate = "id = 3" + print("Selected:", table.search().where(predicate).to_list()) + table.delete(predicate) + rows = sorted(table.search().to_list(), key=lambda row: row["id"]) + print("Remaining:", rows) + # --8<-- [end:delete_rows] + + assert rows == [{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}] + + +def test_add_progress(): + # --8<-- [start:add_progress] + from tempfile import TemporaryDirectory + + import lancedb + + def report_progress(progress): + print(f"Rows written: {progress['output_rows']}") + + with TemporaryDirectory(prefix="lancedb-ingestion-") as uri: + db = lancedb.connect(uri) + table = db.create_table("users", [{"id": 1, "name": "Alice"}]) + table.add( + [{"id": 2, "name": "Bob"}, {"id": 3, "name": "Charlie"}], + progress=report_progress, + ) + rows = sorted(table.search().to_list(), key=lambda row: row["id"]) + print("After insertion:", rows) + # --8<-- [end:add_progress] + + assert rows == [ + {"id": 1, "name": "Alice"}, + {"id": 2, "name": "Bob"}, + {"id": 3, "name": "Charlie"}, + ] diff --git a/docs/web-tests/py/test_integrations.py b/docs/web-tests/py/test_integrations.py index ff07dfae0..577847504 100644 --- a/docs/web-tests/py/test_integrations.py +++ b/docs/web-tests/py/test_integrations.py @@ -740,13 +740,16 @@ def test_reranking_answerdotai_usage() -> None: require_flag("RUN_RERANKER_SNIPPETS") # --8<-- [start:reranking_answerdotai_usage] + import tempfile + from pathlib import Path + import lancedb from lancedb.embeddings import get_registry from lancedb.pydantic import LanceModel, Vector from lancedb.rerankers import AnswerdotaiRerankers embedder = get_registry().get("sentence-transformers").create() - db = lancedb.connect("~/.lancedb") + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "answerdotai-rerank")) class Schema(LanceModel): text: str = embedder.SourceField() @@ -756,18 +759,20 @@ def test_reranking_answerdotai_usage() -> None: {"text": "hello world"}, {"text": "goodbye world"}, ] - tbl = db.create_table("test", schema=Schema, mode="overwrite") + tbl = db.create_table("test", schema=Schema) tbl.add(data) reranker = AnswerdotaiRerankers() # Run vector search with a reranker result = tbl.search("hello").rerank(reranker=reranker).to_list() + # Build the full-text index before either FTS or hybrid search. + tbl.create_fts_index("text", replace=True) + # Run FTS search with a reranker result = tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list() # Run hybrid search with a reranker - tbl.create_fts_index("text", replace=True) result = ( tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() ) @@ -819,13 +824,16 @@ def test_reranking_colbert_usage() -> None: require_flag("RUN_RERANKER_SNIPPETS") # --8<-- [start:reranking_colbert_usage] + import tempfile + from pathlib import Path + import lancedb from lancedb.embeddings import get_registry from lancedb.pydantic import LanceModel, Vector from lancedb.rerankers import ColbertReranker embedder = get_registry().get("sentence-transformers").create() - db = lancedb.connect("~/.lancedb") + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "colbert-rerank")) class Schema(LanceModel): text: str = embedder.SourceField() @@ -835,18 +843,20 @@ def test_reranking_colbert_usage() -> None: {"text": "hello world"}, {"text": "goodbye world"}, ] - tbl = db.create_table("test", schema=Schema, mode="overwrite") + tbl = db.create_table("test", schema=Schema) tbl.add(data) reranker = ColbertReranker() # Run vector search with a reranker result = tbl.search("hello").rerank(reranker=reranker).to_list() + # Build the full-text index before either FTS or hybrid search. + tbl.create_fts_index("text", replace=True) + # Run FTS search with a reranker result = tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list() # Run hybrid search with a reranker - tbl.create_fts_index("text", replace=True) result = ( tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() ) @@ -857,13 +867,16 @@ def test_reranking_cross_encoder_usage() -> None: require_flag("RUN_RERANKER_SNIPPETS") # --8<-- [start:reranking_cross_encoder_usage] + import tempfile + from pathlib import Path + import lancedb from lancedb.embeddings import get_registry from lancedb.pydantic import LanceModel, Vector from lancedb.rerankers import CrossEncoderReranker embedder = get_registry().get("sentence-transformers").create() - db = lancedb.connect("~/.lancedb") + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "cross-encoder-rerank")) class Schema(LanceModel): text: str = embedder.SourceField() @@ -873,18 +886,20 @@ def test_reranking_cross_encoder_usage() -> None: {"text": "hello world"}, {"text": "goodbye world"}, ] - tbl = db.create_table("test", schema=Schema, mode="overwrite") + tbl = db.create_table("test", schema=Schema) tbl.add(data) reranker = CrossEncoderReranker() # Run vector search with a reranker result = tbl.search("hello").rerank(reranker=reranker).to_list() + # Build the full-text index before either FTS or hybrid search. + tbl.create_fts_index("text", replace=True) + # Run FTS search with a reranker result = tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list() # Run hybrid search with a reranker - tbl.create_fts_index("text", replace=True) result = ( tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list() ) @@ -936,13 +951,16 @@ def test_reranking_linear_combination_usage() -> None: require_flag("RUN_RERANKER_SNIPPETS") # --8<-- [start:reranking_linear_combination_usage] + import tempfile + from pathlib import Path + import lancedb from lancedb.embeddings import get_registry from lancedb.pydantic import LanceModel, Vector from lancedb.rerankers import LinearCombinationReranker embedder = get_registry().get("sentence-transformers").create() - db = lancedb.connect("~/.lancedb") + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "linear-combination-rerank")) class Schema(LanceModel): text: str = embedder.SourceField() @@ -952,7 +970,7 @@ def test_reranking_linear_combination_usage() -> None: {"text": "hello world"}, {"text": "goodbye world"}, ] - tbl = db.create_table("test", schema=Schema, mode="overwrite") + tbl = db.create_table("test", schema=Schema) tbl.add(data) reranker = LinearCombinationReranker() @@ -1007,13 +1025,16 @@ def test_reranking_rrf_usage() -> None: require_flag("RUN_RERANKER_SNIPPETS") # --8<-- [start:reranking_rrf_usage] + import tempfile + from pathlib import Path + import lancedb from lancedb.embeddings import get_registry from lancedb.pydantic import LanceModel, Vector from lancedb.rerankers import RRFReranker embedder = get_registry().get("sentence-transformers").create() - db = lancedb.connect("~/.lancedb") + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "rrf-rerank")) class Schema(LanceModel): text: str = embedder.SourceField() @@ -1023,7 +1044,7 @@ def test_reranking_rrf_usage() -> None: {"text": "hello world"}, {"text": "goodbye world"}, ] - tbl = db.create_table("test", schema=Schema, mode="overwrite") + tbl = db.create_table("test", schema=Schema) tbl.add(data) reranker = RRFReranker() @@ -1039,13 +1060,16 @@ def test_reranking_mrr_usage() -> None: require_flag("RUN_RERANKER_SNIPPETS") # --8<-- [start:reranking_mrr_usage] + import tempfile + from pathlib import Path + import lancedb from lancedb.embeddings import get_registry from lancedb.pydantic import LanceModel, Vector from lancedb.rerankers import MRRReranker embedder = get_registry().get("sentence-transformers").create() - db = lancedb.connect("~/.lancedb") + db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "mrr-rerank")) class Schema(LanceModel): text: str = embedder.SourceField() @@ -1055,7 +1079,7 @@ def test_reranking_mrr_usage() -> None: {"text": "hello world"}, {"text": "goodbye world"}, ] - tbl = db.create_table("test", schema=Schema, mode="overwrite") + tbl = db.create_table("test", schema=Schema) tbl.add(data) reranker = MRRReranker(weight_vector=0.7, weight_fts=0.3) @@ -1390,9 +1414,12 @@ def test_frameworks_pydantic_examples() -> None: {"text": "goodbye world", "vector": [0.0, 1.0]}, ] ) - results = table.search("hello world").limit(1).to_pydantic(LanceDocs) + results = table.search([1.0, 0.0]).limit(1).to_pydantic(LanceDocs) print(results[0].text) + # hello world # --8<-- [end:frameworks_pydantic_base_example] + assert isinstance(results[0], LanceDocs) + assert results[0].text == "hello world" # Platform integrations @@ -1686,7 +1713,7 @@ def test_platforms_pandas_examples() -> None: # --8<-- [end:platforms_pandas_vector_search] # --8<-- [start:platforms_pandas_async_example] - async def run_pandas_async_example() -> None: + async def run_pandas_async_example() -> pd.DataFrame: async_db = await lancedb.connect_async( str(Path(tempfile.mkdtemp()) / "pandas-async") ) @@ -1699,16 +1726,16 @@ def test_platforms_pandas_examples() -> None: async_table = await async_db.create_table( "creatures_async", data=async_df, mode="overwrite" ) - async_results = await ( - async_table.search([0.6, 0.4, 0.8]) - .select(["text", "_distance"]) - .limit(1) - .to_pandas() - ) + # An async search returns its query builder from an await + query = await async_table.search([0.6, 0.4, 0.8]) + async_results = await query.select(["text", "_distance"]).limit(1).to_pandas() print(async_results) + return async_results - asyncio.run(run_pandas_async_example()) + async_results = asyncio.run(run_pandas_async_example()) # --8<-- [end:platforms_pandas_async_example] + assert async_results["text"].tolist() == ["sage"] + assert async_results["_distance"].tolist() == [0.0] def test_platforms_polars_examples() -> None: diff --git a/docs/web-tests/py/test_materialized_view.py b/docs/web-tests/py/test_materialized_view.py index ec48e1033..414d6dc1e 100644 --- a/docs/web-tests/py/test_materialized_view.py +++ b/docs/web-tests/py/test_materialized_view.py @@ -12,7 +12,9 @@ def test_materialized_view_create_and_refresh(tmp_path, monkeypatch): db = lancedb.connect("./.lancedb") table = db.create_table( - "people", [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}] + "people", + [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}], + storage_options={"new_table_enable_stable_row_ids": "true"}, ) view = db.create_materialized_view( @@ -29,12 +31,11 @@ def test_materialized_view_create_and_refresh(tmp_path, monkeypatch): # --8<-- [start:materialized_view_refresh] table.add([{"name": "bea", "age": 41}]) - result = view.refresh() - print(result.rows_written) + view.refresh() + print(view.table.count_rows()) # 2 # --8<-- [end:materialized_view_refresh] - assert result.mode == "rebuild" - assert result.rows_written == 2 + assert view.table.count_rows() == 2 assert sorted(row["shout"] for row in view.table.to_arrow().to_pylist()) == [ "ADA", "BEA", @@ -45,7 +46,11 @@ def test_materialized_view_with_no_data(tmp_path): import lancedb db = lancedb.connect(tmp_path) - db.create_table("people", [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}]) + db.create_table( + "people", + [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}], + storage_options={"new_table_enable_stable_row_ids": "true"}, + ) view = db.create_materialized_view( "adults", "people", where="age >= 18", with_no_data=True ) @@ -58,11 +63,16 @@ def test_materialized_view_refresh_to_source_version(tmp_path): import lancedb db = lancedb.connect(tmp_path) - table = db.create_table("people", [{"name": "ada", "age": 36}]) + table = db.create_table( + "people", + [{"name": "ada", "age": 36}], + storage_options={"new_table_enable_stable_row_ids": "true"}, + ) view = db.create_materialized_view("adults", "people", where="age >= 18") first = table.version table.add([{"name": "bea", "age": 41}]) - assert view.refresh().rows_written == 2 + view.refresh() + assert view.table.count_rows() == 2 result = view.refresh(source_version=first) assert (result.source_version, result.rows_written) == (first, 1) diff --git a/docs/web-tests/py/test_provider_examples.py b/docs/web-tests/py/test_provider_examples.py new file mode 100644 index 000000000..e1836b6ff --- /dev/null +++ b/docs/web-tests/py/test_provider_examples.py @@ -0,0 +1,160 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +"""The provider reranker examples write only to a directory they create. + +Each example runs offline. A stand-in embedding function answers for the model +the example asks the registry for, a stand-in reranker replaces the provider +client, and outbound connections are refused, so no model is downloaded and no +provider is called. Sentinel tables in the owned home directory, where these +examples used to write, and in the temporary directory must be left unchanged. +""" + +import importlib.util +import socket +import tempfile +from pathlib import Path +from types import SimpleNamespace +from typing import ClassVar + +import lancedb +import lancedb.embeddings +import lancedb.rerankers +import pyarrow as pa +import pytest +from lancedb.embeddings import TextEmbeddingFunction, register +from lancedb.rerankers import Reranker + +EXAMPLES = Path(__file__).with_name("provider_examples.py") +RERANKERS = { + "reranking_cohere_usage": "CohereReranker", + "reranking_jina_usage": "JinaReranker", + "reranking_watsonx_usage": "WatsonxReranker", + "reranking_openai_usage": "OpenaiReranker", + "reranking_voyageai_usage": "VoyageAIReranker", +} + + +@register("docs-offline-text") +class OfflineEmbedding(TextEmbeddingFunction): + """Deterministic four-dimensional vectors, computed locally.""" + + def ndims(self): + return 4 + + def generate_embeddings(self, texts): + return [[float(len(text)), 1.0, 0.0, 0.0] for text in texts] + + +class OfflineReranker(Reranker): + """Scores every row 1.0 and records which searches it reranked.""" + + instances: ClassVar[list["OfflineReranker"]] = [] + + def __init__(self, **kwargs): + super().__init__() + self.kwargs = kwargs + self.searches = [] + OfflineReranker.instances.append(self) + + def _score(self, results, search): + self.searches.append(search) + drop = [c for c in ("_distance", "_score") if c in results.column_names] + results = results.drop_columns(drop) + scores = pa.array([1.0] * results.num_rows, pa.float32()) + return results.append_column("_relevance_score", scores) + + def rerank_vector(self, query, vector_results): + return self._score(vector_results, "vector") + + def rerank_fts(self, query, fts_results): + return self._score(fts_results, "fts") + + def rerank_hybrid(self, query, vector_results, fts_results): + merged = self.merge_results(vector_results, fts_results) + return self._score(merged, "hybrid") + + +def snapshot(connect, uri): + db = connect(str(uri)) + tables = {} + for name in db.list_tables().tables: + table = db.open_table(name) + tables[name] = (table.version, table.schema, table.to_arrow().to_pylist()) + return tables + + +def load_examples(): + spec = importlib.util.spec_from_file_location("provider_examples", EXAMPLES) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def owned(tmp_path, monkeypatch): + home, temp_root = tmp_path / "home", tmp_path / "tmp" + home.mkdir() + temp_root.mkdir() + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setattr(tempfile, "tempdir", str(temp_root)) + monkeypatch.chdir(tmp_path) + # Anything the examples resolve from "~" must land in the owned home. + assert Path("~/.lancedb").expanduser().resolve().is_relative_to(home.resolve()) + + def refuse(*args, **kwargs): + raise OSError("network access is disabled in this test") + + monkeypatch.setattr(socket.socket, "connect", refuse) + monkeypatch.setattr(socket, "create_connection", refuse) + + real_connect = lancedb.connect + sentinels = [home / ".lancedb", temp_root / "existing"] + for uri in sentinels: + row = {"text": "keep me", "vector": [0.5, 0.5, 0.5, 0.5]} + real_connect(str(uri)).create_table("test", [row]) + before = [snapshot(real_connect, uri) for uri in sentinels] + + registry = lancedb.embeddings.get_registry() + monkeypatch.setattr(registry, "get", lambda name: OfflineEmbedding) + for name in RERANKERS.values(): + monkeypatch.setattr(lancedb.rerankers, name, OfflineReranker) + monkeypatch.setenv("JINA_API_KEY", "replaced-by-the-example") + OfflineReranker.instances.clear() + + connected = [] + + def recording_connect(uri, *args, **kwargs): + connected.append(Path(uri).expanduser().resolve()) + return real_connect(uri, *args, **kwargs) + + monkeypatch.setattr(lancedb, "connect", recording_connect) + return SimpleNamespace( + home=home, + temp_root=temp_root, + sentinels=sentinels, + before=before, + connect=real_connect, + connected=connected, + ) + + +@pytest.mark.parametrize("example", sorted(RERANKERS)) +def test_reranker_example_writes_only_to_its_own_directory(owned, example): + existing = set(owned.temp_root.iterdir()) + + getattr(load_examples(), example)() + + after = [snapshot(owned.connect, uri) for uri in owned.sentinels] + assert after == owned.before + created = set(owned.temp_root.iterdir()) - existing + assert len(created) == 1 + (own_directory,) = created + assert len(owned.connected) == 1 + assert owned.connected[0].is_relative_to(own_directory.resolve()) + assert owned.connect(str(owned.connected[0])).list_tables().tables == ["test"] + + (reranker,) = OfflineReranker.instances + assert reranker.searches == ["vector", "fts", "hybrid"] + assert reranker.kwargs["api_key"] == "your-api-key" + assert reranker.kwargs.get("project_id", "your-project-id") == "your-project-id" diff --git a/docs/web-tests/py/test_quickstart.py b/docs/web-tests/py/test_quickstart.py index ee01e7317..779a268d4 100644 --- a/docs/web-tests/py/test_quickstart.py +++ b/docs/web-tests/py/test_quickstart.py @@ -1,14 +1,125 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright The LanceDB Authors +import os +import shutil +import uuid +from pathlib import Path + import lancedb import pytest +import pytest_asyncio + +# What the quickstart's download step leaves in an application's directory. +IMAGE = ( + Path(__file__).resolve().parents[2] + / "web/static/assets/images/quickstart/sir-lancelot.jpg" +) +DEPLOYMENT = ("LANCEDB_URI", "LANCEDB_HOST_OVERRIDE", "LANCEDB_API_KEY") +# The connection settings alone never send these tests to a deployment. +OPT_IN = "LANCEDB_DOCS_TEST_DEPLOYMENT" -def test_quickstart(db_path_factory): - uri = db_path_factory("quickstart_db") - db = lancedb.connect(uri) +def deployment_settings(): + if os.environ.get(OPT_IN) != "1": + pytest.skip(f"set {OPT_IN}=1 and {', '.join(DEPLOYMENT)} to use a deployment") + missing = [name for name in DEPLOYMENT if not os.environ.get(name)] + if missing: + pytest.fail(f"{OPT_IN}=1 needs {', '.join(missing)}") + return dict( + uri=os.environ["LANCEDB_URI"], + api_key=os.environ["LANCEDB_API_KEY"], + region=os.environ.get("LANCEDB_REGION", "us-east-1"), + host_override=os.environ["LANCEDB_HOST_OVERRIDE"], + ) + +class OwnedTables: + """A deployment connection on which a test creates only tables of its own. + + Each name gets a suffix unique to the test and is claimed only while no + table has it; the test drops what it claimed, whether or not it passed. + """ + + def __init__(self, db): + self.db = db + self.suffix = uuid.uuid4().hex[:12] + self.claimed = [] + + def claim(self, name, existing, mode): + owned = f"{name}_{self.suffix}" + if owned in self.claimed: + return owned, mode + if owned in existing: + raise RuntimeError(f"{owned} already exists and is not this test's") + self.claimed.append(owned) + return owned, "create" + + def table_names(self): + names, token = [], None + while True: + page = self.db.list_tables(page_token=token) + names += page.tables + if not (token := page.page_token): + return names + + def create_table(self, name, *args, mode="create", **kwargs): + owned, mode = self.claim(name, self.table_names(), mode) + return self.db.create_table(owned, *args, mode=mode, **kwargs) + + def drop_claimed(self): + existing = self.table_names() + for name in self.claimed: + if name in existing: + self.db.drop_table(name) + + +class AsyncOwnedTables(OwnedTables): + async def table_names(self): + names, token = [], None + while True: + page = await self.db.list_tables(page_token=token) + names += page.tables + if not (token := page.page_token): + return names + + async def create_table(self, name, *args, mode="create", **kwargs): + owned, mode = self.claim(name, await self.table_names(), mode) + return await self.db.create_table(owned, *args, mode=mode, **kwargs) + + async def drop_claimed(self): + existing = await self.table_names() + for name in self.claimed: + if name in existing: + await self.db.drop_table(name) + + +@pytest.fixture +def deployment(): + owned = OwnedTables(lancedb.connect(**deployment_settings())) + yield owned + owned.drop_claimed() + + +@pytest.fixture(params=["local", "enterprise"]) +def db(request, db_path_factory): + if request.param == "local": + yield lancedb.connect(db_path_factory("quickstart_db")) + else: + yield request.getfixturevalue("deployment") + + +@pytest_asyncio.fixture(params=["local", "enterprise"]) +async def async_db(request, db_path_factory): + if request.param == "local": + yield await lancedb.connect_async(db_path_factory("quickstart_async_db")) + return + owned = AsyncOwnedTables(await lancedb.connect_async(**deployment_settings())) + yield owned + await owned.drop_claimed() + + +def create_characters(db): # --8<-- [start:quickstart_data] data = [ { @@ -41,14 +152,29 @@ def test_quickstart(db_path_factory): # --8<-- [start:quickstart_create_table] table = db.create_table("characters", data=data, mode="overwrite") # --8<-- [end:quickstart_create_table] - assert len(table) == 3 - # Drop the table to test create without overwrite - db.drop_table("characters") + assert table.count_rows() == 3 + + # --8<-- [start:quickstart_add_data] + table.add( + [ + { + "id": "4", + "name": "Morgana", + "role": "Sorceress", + "description": "Powerful sorceress of Avalon.", + "stats": {"strength": 2, "magic": 5, "leadership": 4, "wisdom": 4}, + "vector": [0.3, 0.9, 0.6, 0.8], + } + ] + ) + # --8<-- [end:quickstart_add_data] + assert table.count_rows() == 4 + return table + + +def test_quickstart(db): + table = create_characters(db) - # --8<-- [start:quickstart_create_table_no_overwrite] - table = db.create_table("characters", data=data) - # --8<-- [end:quickstart_create_table_no_overwrite] - assert len(table) == 3 # --8<-- [start:quickstart_vector_search_1] # Search for examples similar to a "wise magical advisor" query_vector = [0.2, 0.8, 0.4, 0.9] @@ -62,7 +188,14 @@ def test_quickstart(db_path_factory): ) print(result) # --8<-- [end:quickstart_vector_search_1] - assert result.head(1)["name"][0] == "Merlin" + assert result["name"].to_list() == ["Merlin", "Morgana"] + + # --8<-- [start:quickstart_output_pandas] + # Ensure you run `pip install pandas` beforehand + result = table.search(query_vector).limit(2).to_pandas() + print(result) + # --8<-- [end:quickstart_output_pandas] + assert result["name"].to_list() == ["Merlin", "Morgana"] # --8<-- [start:quickstart_curate_with_metadata] curated = ( @@ -74,131 +207,15 @@ def test_quickstart(db_path_factory): ) print(curated) # --8<-- [end:quickstart_curate_with_metadata] - assert curated.head(1)["name"][0] == "Merlin" + assert curated["name"].to_list() == ["Merlin", "Morgana"] - # --8<-- [start:quickstart_output_pandas] - # Ensure you run `pip install pandas` beforehand - result = table.search(query_vector).limit(2).to_pandas() - print(result) - # --8<-- [end:quickstart_output_pandas] - assert result.iloc[0]["name"] == "Merlin" - - # --8<-- [start:quickstart_add_feature] - table.add_columns( - { - "power_score": "cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)" - } - ) - # --8<-- [end:quickstart_add_feature] - assert "power_score" in table.schema.names - - # --8<-- [start:quickstart_query_feature] - features = table.search().select(["name", "role", "power_score"]).to_polars() - print(features) - # --8<-- [end:quickstart_query_feature] - assert "power_score" in features.columns - - # --8<-- [start:quickstart_multimodal_bytes] - from pathlib import Path - - image_path = Path("docs/web/static/assets/images/quickstart/sir-lancelot.jpg") - image_bytes = image_path.read_bytes() - - multimodal_table = db.create_table( - "character_images", - data=[ - { - "id": "lancelot", - "description": "Portrait of Sir Lancelot", - "image": image_bytes, - "vector": [0.9, 0.1, 0.5, 0.4], - } - ], - mode="overwrite", - ) - # --8<-- [end:quickstart_multimodal_bytes] - assert len(multimodal_table) == 1 - - # --8<-- [start:quickstart_open_table] - table = db.open_table("characters") - # --8<-- [end:quickstart_open_table] - - # --8<-- [start:quickstart_add_data] - more_data = [ - { - "id": "4", - "name": "Morgana", - "role": "Sorceress", - "description": "Powerful sorceress of Avalon.", - "stats": {"strength": 2, "magic": 5, "leadership": 4, "wisdom": 4}, - "vector": [0.3, 0.9, 0.6, 0.8], - "power_score": 3.75, - }, - ] - - # Add data to table - table.add(more_data) - # --8<-- [end:quickstart_add_data] - assert len(table) == 4 - - # --8<-- [start:quickstart_vector_search_2] - # Search for examples similar to a "powerful sorceress" - query_vector = [0.3, 0.9, 0.6, 0.8] - - results = table.search(query_vector).limit(2).to_polars() - print(results) - # --8<-- [end:quickstart_vector_search_2] - assert results.head(1)["name"][0] == "Morgana" - - # --8<-- [start:quickstart_computed_feature] - table.add_columns(computed={"combat_score": "(stats.magic + stats.strength) / 2.0"}) - table.refresh_column("combat_score") - # --8<-- [end:quickstart_computed_feature] - scores = ( - table.search().select(["combat_score"]).to_arrow()["combat_score"].to_pylist() - ) - assert scores == [2.5, 3.5, 3.0, 3.5] - - # --8<-- [start:quickstart_training] - import torch - from lancedb.permutation import Permutation - - dataset = Permutation.identity(table).select_columns( - ["power_score", "combat_score"] - ) - dataloader = torch.utils.data.DataLoader(dataset, batch_size=1024, shuffle=True) - # --8<-- [end:quickstart_training] - batches = list(dataloader) - assert len(batches) == 1 - assert set(batches[0]) == {"power_score", "combat_score"} - assert all(isinstance(value, torch.Tensor) for value in batches[0].values()) - assert sorted(batches[0]["combat_score"].tolist()) == sorted(scores) - - # --8<-- [start:quickstart_training_projection] - dataset = Permutation.identity(table).select_columns(["power_score"]) - dataloader = torch.utils.data.DataLoader(dataset, batch_size=1024) - # --8<-- [end:quickstart_training_projection] - batch = next(iter(dataloader)) - assert set(batch) == {"power_score"} - assert batch["power_score"].tolist() == [3.5, 4.0, 3.0, 3.75] + curated = table.search(query_vector).where("stats.magic < 4").limit(2).to_list() + assert [row["name"] for row in curated] == ["King Arthur", "Sir Lancelot"] @pytest.mark.asyncio -async def test_quickstart_async_api(db_path_factory): - db_uri = db_path_factory("quickstart_async_db") - import lancedb - async_db = await lancedb.connect_async(db_uri) - - # --8<-- [start:quickstart_data_async] +async def test_quickstart_async_api(async_db): data = [ - { - "id": "1", - "name": "King Arthur", - "role": "King", - "description": "Leader of Camelot and wielder of Excalibur.", - "stats": {"strength": 4, "magic": 1, "leadership": 5, "wisdom": 4}, - "vector": [0.7, 0.1, 0.9, 0.7], - }, { "id": "2", "name": "Merlin", @@ -206,17 +223,8 @@ async def test_quickstart_async_api(db_path_factory): "description": "Advisor and prophet with deep magical knowledge.", "stats": {"strength": 2, "magic": 5, "leadership": 4, "wisdom": 5}, "vector": [0.2, 0.9, 0.4, 0.9], - }, - { - "id": "3", - "name": "Sir Lancelot", - "role": "Knight", - "description": "Legendary knight known for courage and combat skill.", - "stats": {"strength": 5, "magic": 1, "leadership": 3, "wisdom": 3}, - "vector": [0.9, 0.1, 0.5, 0.4], - }, + } ] - # --8<-- [end:quickstart_data_async] # --8<-- [start:quickstart_create_table_async] async_table = await async_db.create_table( @@ -236,5 +244,110 @@ async def test_quickstart_async_api(db_path_factory): ).select(["name", "role", "description", "_distance"]).limit(2).to_polars() print(async_result) # --8<-- [end:quickstart_vector_search_1_async] + assert async_result["name"].to_list() == ["Merlin"] - assert async_result.head(1)["name"][0] == "Merlin" + +def test_quickstart_derived_feature(db): + table = create_characters(db) + + # --8<-- [start:quickstart_add_feature] + table.add_columns( + { + "power_score": "cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)" + } + ) + # --8<-- [end:quickstart_add_feature] + + # --8<-- [start:quickstart_query_feature] + features = table.search().select(["name", "role", "power_score"]).to_polars() + print(features) + # --8<-- [end:quickstart_query_feature] + assert sorted(zip(features["name"], features["power_score"])) == [ + ("King Arthur", 3.5), + ("Merlin", 4.0), + ("Morgana", 3.75), + ("Sir Lancelot", 3.0), + ] + + +def description_lengths(table): + rows = table.search().select(["description", "description_length"]).to_list() + return {row["description"]: row["description_length"] for row in rows} + + +def test_quickstart_computed_column(db_path_factory): + table = create_characters(lancedb.connect(db_path_factory("computed"))) + + # --8<-- [start:quickstart_computed_feature] + table.add_columns(computed={"description_length": "length(description)"}) + table.refresh_column("description_length") + # --8<-- [end:quickstart_computed_feature] + lengths = description_lengths(table) + assert lengths == {text: len(text) for text in lengths} + + +def test_quickstart_computed_column_on_a_deployment(deployment): + table = create_characters(deployment) + + # --8<-- [start:quickstart_computed_feature_enterprise] + table.add_columns(computed={"description_length": "length(description)"}) + job = table.refresh_column_async("description_length") + job.wait() + # --8<-- [end:quickstart_computed_feature_enterprise] + lengths = description_lengths(table) + assert len(lengths) == 4 + assert lengths == {text: len(text) for text in lengths} + with pytest.raises(Exception): + table.refresh_column("description_length") + + +def test_quickstart_multimodal(db, tmp_path, monkeypatch): + app = tmp_path / "app" + (app / "data").mkdir(parents=True) + shutil.copy(IMAGE, app / "data/sir-lancelot.jpg") + monkeypatch.chdir(app) + + # --8<-- [start:quickstart_multimodal_bytes] + from pathlib import Path + + image_bytes = Path("data/sir-lancelot.jpg").read_bytes() + + multimodal_table = db.create_table( + "character_images", + data=[ + { + "id": "lancelot", + "description": "Portrait of Sir Lancelot", + "image": image_bytes, + "vector": [0.9, 0.1, 0.5, 0.4], + } + ], + mode="overwrite", + ) + # --8<-- [end:quickstart_multimodal_bytes] + assert multimodal_table.count_rows() == 1 + stored = multimodal_table.search().select(["image"]).to_list()[0]["image"] + assert stored == IMAGE.read_bytes() + + +def test_quickstart_training(db): + table = create_characters(db) + + # --8<-- [start:quickstart_training] + import torch + from lancedb.permutation import Permutation + + table.add_columns( + { + "magic": "cast(stats.magic as float)", + "strength": "cast(stats.strength as float)", + } + ) + dataset = Permutation.identity(table).select_columns(["magic", "strength"]) + dataloader = torch.utils.data.DataLoader(dataset, batch_size=1024, shuffle=True) + # --8<-- [end:quickstart_training] + batches = list(dataloader) + assert len(batches) == 1 + assert set(batches[0]) == {"magic", "strength"} + assert all(isinstance(value, torch.Tensor) for value in batches[0].values()) + assert sorted(batches[0]["magic"].tolist()) == [1.0, 1.0, 5.0, 5.0] diff --git a/docs/web-tests/py/test_read_consistency.py b/docs/web-tests/py/test_read_consistency.py new file mode 100644 index 000000000..886bb5676 --- /dev/null +++ b/docs/web-tests/py/test_read_consistency.py @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + + +def test_read_consistency(): + # --8<-- [start:read_consistency] + from datetime import timedelta + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-consistency-") as uri: + writer = lancedb.connect(uri) + table = writer.create_table("documents", [{"id": 1, "text": "first"}]) + + default_reader = lancedb.connect(uri).open_table("documents") + latest_reader = lancedb.connect( + uri, read_consistency_interval=timedelta(0) + ).open_table("documents") + + table.add([{"id": 2, "text": "second"}]) + + default_count = default_reader.count_rows() + latest_count = latest_reader.count_rows() + print(f"Default reader: {default_count} rows") # 1 + print(f"Zero-interval reader: {latest_count} rows") # 2 + + default_reader.checkout_latest() + refreshed_count = default_reader.count_rows() + print(f"After explicit refresh: {refreshed_count} rows") # 2 + # --8<-- [end:read_consistency] + + assert default_count == 1 + assert latest_count == 2 + assert refreshed_count == 2 diff --git a/docs/web-tests/py/test_reranker_examples.py b/docs/web-tests/py/test_reranker_examples.py new file mode 100644 index 000000000..71505d423 --- /dev/null +++ b/docs/web-tests/py/test_reranker_examples.py @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +"""The published reranker examples in test_integrations.py write only to a directory they create. + +Each example runs as readers see it: its marked region is extracted the way the snippet +generator extracts it, saved as a script and run offline. A stand-in embedding function answers for the +sentence-transformers model, stand-in rerankers replace the model-backed AnswerDotAI, ColBERT +and cross-encoder rerankers, and outbound connections are refused. The RRF, linear-combination +and MRR rerankers need no model and run unchanged. Sentinel tables in the owned home directory, +where these examples used to write, and in the temporary directory must be left unchanged. +""" + +import importlib.util +import runpy +import sys +from pathlib import Path + +import lancedb.rerankers +import pytest +import test_provider_examples as provider + +owned = provider.owned +SOURCE = Path(__file__).with_name("test_integrations.py") +GENERATOR = Path(__file__).parents[1] / "mdx_snippets_gen.py" +MODEL_RERANKERS = { + "reranking_answerdotai_usage": "AnswerdotaiRerankers", + "reranking_colbert_usage": "ColbertReranker", + "reranking_cross_encoder_usage": "CrossEncoderReranker", +} +LOCAL_RERANKERS = [ + "reranking_linear_combination_usage", + "reranking_mrr_usage", + "reranking_rrf_usage", +] + + +def load_generator(): + if "mdx_snippets_gen" not in sys.modules: + spec = importlib.util.spec_from_file_location("mdx_snippets_gen", GENERATOR) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module # its dataclasses resolve annotations by module + spec.loader.exec_module(module) + return sys.modules["mdx_snippets_gen"] + + +def reader_code(name): + """The example exactly as the snippet generator publishes it.""" + generator = load_generator() + start, end = generator.markers_for_lang("py") + lines = SOURCE.read_text().splitlines(keepends=True) + snippet = generator.parse_snippets(lines, start, end)[name] + return generator.snippet_text(generator.dedent(snippet)) + + +def run_example(owned, tmp_path, name): + script = tmp_path / "scripts" / f"{name}.py" + script.parent.mkdir(exist_ok=True) + script.write_text(reader_code(name)) + existing = set(owned.temp_root.iterdir()) + namespace = runpy.run_path(str(script)) + + after = [provider.snapshot(owned.connect, uri) for uri in owned.sentinels] + assert after == owned.before + created = set(owned.temp_root.iterdir()) - existing + assert len(created) == 1 + (own_directory,) = created + assert len(owned.connected) == 1 + assert owned.connected[0].is_relative_to(own_directory.resolve()) + assert owned.connect(str(owned.connected[0])).list_tables().tables == ["test"] + return namespace + + +@pytest.mark.parametrize("name", sorted(MODEL_RERANKERS)) +def test_model_reranker_example_writes_only_to_its_own_directory( + owned, monkeypatch, tmp_path, name +): + monkeypatch.setattr( + lancedb.rerankers, MODEL_RERANKERS[name], provider.OfflineReranker + ) + + namespace = run_example(owned, tmp_path, name) + + (reranker,) = provider.OfflineReranker.instances + assert reranker.searches == ["vector", "fts", "hybrid"] + assert namespace["result"] + + +@pytest.mark.parametrize("name", LOCAL_RERANKERS) +def test_local_reranker_example_writes_only_to_its_own_directory(owned, tmp_path, name): + namespace = run_example(owned, tmp_path, name) + + assert namespace["result"] + assert all("_relevance_score" in row for row in namespace["result"]) + if name == "reranking_mrr_usage": + assert namespace["combined"].num_rows > 0 diff --git a/docs/web-tests/py/test_search_examples.py b/docs/web-tests/py/test_search_examples.py new file mode 100644 index 000000000..241a0f771 --- /dev/null +++ b/docs/web-tests/py/test_search_examples.py @@ -0,0 +1,228 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + + +def test_vector_search(): + # --8<-- [start:vector_search] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-search-") as uri: + db = lancedb.connect(uri) + table = db.create_table( + "documents", + [ + { + "id": 1, + "text": "A puppy playing in the park", + "category": "animals", + "vector": [1.0, 0.0], + }, + { + "id": 2, + "text": "A dog resting at home", + "category": "animals", + "vector": [0.9, 0.1], + }, + { + "id": 3, + "text": "A guide to growing tomatoes", + "category": "gardening", + "vector": [0.0, 1.0], + }, + ], + ) + rows = ( + table.search([1.0, 0.0]) + .distance_type("l2") + .select(["id", "text", "_distance"]) + .limit(2) + .to_list() + ) + print(rows) + # --8<-- [end:vector_search] + + assert [row["id"] for row in rows] == [1, 2] + assert rows[0]["_distance"] == 0.0 + + +def test_text_search(): + # --8<-- [start:text_search] + from tempfile import TemporaryDirectory + + import lancedb + from lancedb.index import FTS + + with TemporaryDirectory(prefix="lancedb-search-") as uri: + db = lancedb.connect(uri) + table = db.create_table( + "documents", + [ + { + "id": 1, + "text": "A puppy playing in the park", + "category": "animals", + }, + { + "id": 2, + "text": "A dog resting at home", + "category": "animals", + }, + { + "id": 3, + "text": "A guide to growing tomatoes", + "category": "gardening", + }, + ], + ) + table.create_index("text", config=FTS()) + rows = ( + table.search("puppy", query_type="fts", fts_columns="text") + .select(["id", "text", "_score"]) + .limit(2) + .to_list() + ) + print(rows) + # --8<-- [end:text_search] + + assert [row["id"] for row in rows] == [1] + assert rows[0]["_score"] > 0 + + +def test_hybrid_search(): + # --8<-- [start:hybrid_search] + from tempfile import TemporaryDirectory + + import lancedb + from lancedb.index import FTS + + with TemporaryDirectory(prefix="lancedb-search-") as uri: + db = lancedb.connect(uri) + table = db.create_table( + "documents", + [ + { + "id": 1, + "text": "A puppy playing in the park", + "category": "animals", + "vector": [1.0, 0.0], + }, + { + "id": 2, + "text": "A dog resting at home", + "category": "animals", + "vector": [0.9, 0.1], + }, + { + "id": 3, + "text": "A guide to growing tomatoes", + "category": "gardening", + "vector": [0.0, 1.0], + }, + ], + ) + from lancedb.rerankers import RRFReranker + + table.create_index("text", config=FTS()) + rows = ( + table.search( + query_type="hybrid", vector_column_name="vector", fts_columns="text" + ) + .vector([1.0, 0.0]) + .text("puppy") + .rerank(RRFReranker()) + .limit(2) + .to_list() + ) + print(rows) + # --8<-- [end:hybrid_search] + + assert [row["id"] for row in rows] == [1, 2] + assert rows[0]["_relevance_score"] > rows[1]["_relevance_score"] + + +def test_filter_search(): + # --8<-- [start:filter_search] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-search-") as uri: + db = lancedb.connect(uri) + table = db.create_table( + "documents", + [ + { + "id": 1, + "text": "A puppy playing in the park", + "category": "animals", + "vector": [1.0, 0.0], + }, + { + "id": 2, + "text": "A dog resting at home", + "category": "animals", + "vector": [0.9, 0.1], + }, + { + "id": 3, + "text": "A guide to growing tomatoes", + "category": "gardening", + "vector": [0.0, 1.0], + }, + ], + ) + rows = ( + table.search([0.0, 1.0]) + .where("category = 'animals'", prefilter=True) + .select(["id", "text", "category", "_distance"]) + .limit(2) + .to_list() + ) + print(rows) + # --8<-- [end:filter_search] + + assert {row["id"] for row in rows} == {1, 2} + assert all(row["category"] == "animals" for row in rows) + + +def test_query_plan(): + # --8<-- [start:query_plan] + from tempfile import TemporaryDirectory + + import lancedb + + with TemporaryDirectory(prefix="lancedb-search-") as uri: + db = lancedb.connect(uri) + table = db.create_table( + "documents", + [ + { + "id": 1, + "text": "A puppy playing in the park", + "category": "animals", + "vector": [1.0, 0.0], + }, + { + "id": 2, + "text": "A dog resting at home", + "category": "animals", + "vector": [0.9, 0.1], + }, + { + "id": 3, + "text": "A guide to growing tomatoes", + "category": "gardening", + "vector": [0.0, 1.0], + }, + ], + ) + query = table.search([1.0, 0.0]).select(["id", "text", "_distance"]).limit(2) + print("Plan:", query.explain_plan()) + print("Measured execution:", query.analyze_plan()) + rows = query.to_list() + print(rows) + # --8<-- [end:query_plan] + + assert [row["id"] for row in rows] == [1, 2] diff --git a/docs/web-tests/py/test_tables.py b/docs/web-tests/py/test_tables.py index b3a456adc..d17344aec 100644 --- a/docs/web-tests/py/test_tables.py +++ b/docs/web-tests/py/test_tables.py @@ -835,31 +835,31 @@ def test_update_optimize_cleanup_snippet(tmp_db): # ============================================================================ -def _setup_schema_add_table(tmp_db, data=None): +def _setup_schema_add_table(db): # --8<-- [start:schema_add_setup] - table_name = "schema_evolution_add_example" - if data is None: - data = [ - { - "id": 1, - "name": "Laptop", - "price": 1200.00, - "vector": np.random.random(128).tolist(), - }, - { - "id": 2, - "name": "Smartphone", - "price": 800.00, - "vector": np.random.random(128).tolist(), - }, - { - "id": 3, - "name": "Headphones", - "price": 150.00, - "vector": np.random.random(128).tolist(), - }, - ] - table = tmp_db.create_table(table_name, data, mode="overwrite") + import numpy as np + + data = [ + { + "id": 1, + "name": "Laptop", + "price": 1200.00, + "vector": np.random.random(128).tolist(), + }, + { + "id": 2, + "name": "Smartphone", + "price": 800.00, + "vector": np.random.random(128).tolist(), + }, + { + "id": 3, + "name": "Headphones", + "price": 150.00, + "vector": np.random.random(128).tolist(), + }, + ] + table = db.create_table("schema_evolution_add_example", data, mode="overwrite") # --8<-- [end:schema_add_setup] return table @@ -953,17 +953,7 @@ def test_add_columns_default_values(tmp_db): def test_add_columns_nullable(tmp_db): - table = _setup_schema_add_table( - tmp_db, - data=[ - { - "id": 1, - "name": "Laptop", - "price": 1200.00, - "vector": np.random.random(128).tolist(), - } - ], - ) + table = _setup_schema_add_table(tmp_db) # --8<-- [start:add_columns_nullable] # Add a nullable timestamp column diff --git a/docs/web-tests/rs/Cargo.lock b/docs/web-tests/rs/Cargo.lock index 599633c5e..447bae7f6 100644 --- a/docs/web-tests/rs/Cargo.lock +++ b/docs/web-tests/rs/Cargo.lock @@ -452,6 +452,61 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "backtrace" version = "0.3.76" @@ -845,6 +900,16 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -1769,26 +1834,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "deepsize" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cdb987ec36f6bf7bfbea3f928b75590b736fc42af8e54d97592481351b2b96c" -dependencies = [ - "deepsize_derive", -] - -[[package]] -name = "deepsize_derive" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990101d41f3bc8c1a45641024377ee284ecc338e5ecf3ea0f0e236d897c72796" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "deranged" version = "0.5.5" @@ -2416,6 +2461,12 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "humantime" version = "2.3.0" @@ -2436,6 +2487,7 @@ dependencies = [ "http", "http-body", "httparse", + "httpdate", "itoa", "pin-project-lite", "pin-utils", @@ -2480,9 +2532,11 @@ dependencies = [ "percent-encoding", "pin-project-lite", "socket2", + "system-configuration", "tokio", "tower-service", "tracing", + "windows-registry", ] [[package]] @@ -2938,15 +2992,15 @@ dependencies = [ "half", "humantime", "itertools 0.13.0", - "lance-arrow 8.0.0", - "lance-core 8.0.0", + "lance-arrow", + "lance-core", "lance-datafusion", "lance-encoding", "lance-file", "lance-index", "lance-io", "lance-linalg", - "lance-namespace 8.0.0", + "lance-namespace", "lance-select", "lance-table", "lance-tokenizer", @@ -2974,28 +3028,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "lance-arrow" -version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "253f4a0a70580c985b91e65e9ca6cad644825a4078de28d8efbacf3ffbd7ecdc" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-ipc", - "arrow-ord", - "arrow-schema", - "arrow-select", - "bytes", - "futures", - "getrandom 0.2.16", - "half", - "jsonb", - "num-traits", - "rand 0.9.5", -] - [[package]] name = "lance-arrow" version = "8.0.0" @@ -3056,41 +3088,6 @@ dependencies = [ "seq-macro", ] -[[package]] -name = "lance-core" -version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13f84020da5a484e2f07dd1796e09785ed7cd889857ebc4cb77e32ef214ee594" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-schema", - "async-trait", - "byteorder", - "bytes", - "deepsize", - "futures", - "itertools 0.13.0", - "lance-arrow 7.0.0", - "libc", - "log", - "moka", - "num_cpus", - "object_store", - "pin-project", - "prost", - "rand 0.9.5", - "roaring", - "serde_json", - "snafu 0.9.1", - "tempfile", - "tokio", - "tokio-stream", - "tokio-util", - "tracing", - "url", -] - [[package]] name = "lance-core" version = "8.0.0" @@ -3108,7 +3105,7 @@ dependencies = [ "datafusion-sql", "futures", "itertools 0.13.0", - "lance-arrow 8.0.0", + "lance-arrow", "lance-derive", "libc", "libm", @@ -3152,8 +3149,8 @@ dependencies = [ "datafusion-physical-expr", "futures", "jsonb", - "lance-arrow 8.0.0", - "lance-core 8.0.0", + "lance-arrow", + "lance-core", "lance-datagen", "log", "pin-project", @@ -3214,9 +3211,9 @@ dependencies = [ "hex", "hyperloglogplus", "itertools 0.13.0", - "lance-arrow 8.0.0", + "lance-arrow", "lance-bitpacking", - "lance-core 8.0.0", + "lance-core", "log", "lz4", "num-traits", @@ -3248,8 +3245,8 @@ dependencies = [ "bytes", "datafusion-common", "futures", - "lance-arrow 8.0.0", - "lance-core 8.0.0", + "lance-arrow", + "lance-core", "lance-encoding", "lance-io", "log", @@ -3294,9 +3291,9 @@ dependencies = [ "itertools 0.13.0", "jieba-rs", "jsonb", - "lance-arrow 8.0.0", + "lance-arrow", "lance-arrow-stats", - "lance-core 8.0.0", + "lance-core", "lance-datafusion", "lance-datagen", "lance-encoding", @@ -3351,9 +3348,9 @@ dependencies = [ "futures", "http", "io-uring", - "lance-arrow 8.0.0", - "lance-core 8.0.0", - "lance-namespace 8.0.0", + "lance-arrow", + "lance-core", + "lance-namespace", "log", "moka", "object_store", @@ -3379,26 +3376,12 @@ dependencies = [ "arrow-schema", "cc", "half", - "lance-arrow 8.0.0", - "lance-core 8.0.0", + "lance-arrow", + "lance-core", "num-traits", "rand 0.9.5", ] -[[package]] -name = "lance-namespace" -version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "014e8332ca0615506342e0d3af608639864b68396973be14239f09c9f21f1fc2" -dependencies = [ - "arrow", - "async-trait", - "bytes", - "lance-core 7.0.0", - "lance-namespace-reqwest-client 0.7.7", - "snafu 0.9.1", -] - [[package]] name = "lance-namespace" version = "8.0.0" @@ -3408,8 +3391,8 @@ dependencies = [ "arrow", "async-trait", "bytes", - "lance-core 8.0.0", - "lance-namespace-reqwest-client 0.8.6", + "lance-core", + "lance-namespace-reqwest-client", "snafu 0.9.1", ] @@ -3423,42 +3406,33 @@ dependencies = [ "arrow-ipc", "arrow-schema", "async-trait", + "axum", "bytes", "datafusion-common", "datafusion-physical-plan", "futures", "lance", - "lance-core 8.0.0", + "lance-core", "lance-index", "lance-io", "lance-linalg", - "lance-namespace 8.0.0", + "lance-namespace", "lance-table", "log", "object_store", "rand 0.9.5", + "reqwest", "roaring", + "serde", "serde_json", "time", "tokio", + "tower", + "tower-http 0.5.2", "url", "uuid", ] -[[package]] -name = "lance-namespace-reqwest-client" -version = "0.7.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6369eee4682fb11edf538388b43c61ce288b8302fe89bb40944d7daa7faaae99" -dependencies = [ - "reqwest", - "serde", - "serde_json", - "serde_repr", - "serde_with", - "url", -] - [[package]] name = "lance-namespace-reqwest-client" version = "0.8.6" @@ -3485,7 +3459,7 @@ dependencies = [ "byteorder", "bytes", "itertools 0.13.0", - "lance-core 8.0.0", + "lance-core", "roaring", "tracing", ] @@ -3506,8 +3480,8 @@ dependencies = [ "bytes", "chrono", "futures", - "lance-arrow 8.0.0", - "lance-core 8.0.0", + "lance-arrow", + "lance-core", "lance-file", "lance-io", "lance-select", @@ -3538,7 +3512,7 @@ dependencies = [ "arrow-array", "arrow-schema", "criterion", - "lance-arrow 8.0.0", + "lance-arrow", "num-traits", "pprof", "rand 0.9.5", @@ -3589,9 +3563,10 @@ dependencies = [ "datafusion-sql", "futures", "half", + "http", "lance", - "lance-arrow 8.0.0", - "lance-core 8.0.0", + "lance-arrow", + "lance-core", "lance-datafusion", "lance-datagen", "lance-encoding", @@ -3599,7 +3574,7 @@ dependencies = [ "lance-index", "lance-io", "lance-linalg", - "lance-namespace 8.0.0", + "lance-namespace", "lance-namespace-impls", "lance-table", "lance-testing", @@ -3613,6 +3588,7 @@ dependencies = [ "polars-arrow", "rand 0.9.5", "regex", + "reqwest", "semver", "serde", "serde_json", @@ -3621,6 +3597,7 @@ dependencies = [ "tempfile", "tokio", "url", + "urlencoding", "uuid", ] @@ -3867,6 +3844,12 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "matrixmultiply" version = "0.3.10" @@ -5391,7 +5374,7 @@ dependencies = [ "tokio-rustls", "tokio-util", "tower", - "tower-http", + "tower-http 0.6.8", "tower-service", "url", "wasm-bindgen", @@ -5472,7 +5455,8 @@ dependencies = [ "arrow-schema", "futures", "futures-util", - "lance-namespace 7.0.0", + "lance-index", + "lance-namespace", "lancedb", "polars", "polars-arrow", @@ -5659,7 +5643,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" dependencies = [ "bitflags 2.10.0", - "core-foundation", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -5730,6 +5714,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + [[package]] name = "serde_repr" version = "0.1.20" @@ -6203,6 +6198,27 @@ dependencies = [ "windows 0.52.0", ] +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags 2.10.0", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tagptr" version = "0.2.0" @@ -6444,6 +6460,24 @@ dependencies = [ "tokio", "tower-layer", "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" +dependencies = [ + "bitflags 2.10.0", + "bytes", + "http", + "http-body", + "http-body-util", + "pin-project-lite", + "tower-layer", + "tower-service", + "tracing", ] [[package]] @@ -6487,6 +6521,7 @@ version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d15d90a0b5c19378952d479dc858407149d7bb45a14de0142f6c534b16fc647" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -6641,6 +6676,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf8-ranges" version = "1.0.5" @@ -6927,8 +6968,8 @@ dependencies = [ "windows-implement", "windows-interface", "windows-link 0.1.3", - "windows-result", - "windows-strings", + "windows-result 0.3.4", + "windows-strings 0.4.2", ] [[package]] @@ -6986,6 +7027,17 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link 0.2.1", + "windows-result 0.4.1", + "windows-strings 0.5.1", +] + [[package]] name = "windows-result" version = "0.3.4" @@ -6995,6 +7047,15 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link 0.2.1", +] + [[package]] name = "windows-strings" version = "0.4.2" @@ -7004,6 +7065,15 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link 0.2.1", +] + [[package]] name = "windows-sys" version = "0.52.0" diff --git a/docs/web-tests/rs/Cargo.toml b/docs/web-tests/rs/Cargo.toml index c8a4cc7ca..8bc594c3c 100644 --- a/docs/web-tests/rs/Cargo.toml +++ b/docs/web-tests/rs/Cargo.toml @@ -9,8 +9,9 @@ arrow-json = "58.3.0" arrow-schema = "58.3.0" futures = "0.3.31" futures-util = "0.3.31" -lance-namespace = "7.0.0" -lancedb = { version = "0.31.0", features = ["polars"] } +lance-namespace = "8.0.0" +lance-index = "=8.0.0" +lancedb = { version = "0.31.0", features = ["polars", "remote"] } polars = ">=0.37,<0.40.0" polars-arrow = ">=0.37,<0.40.0" serde = { version = "1", features = ["derive"] } @@ -41,3 +42,35 @@ path = "multimodal.rs" [[example]] name = "search" path = "search.rs" + +[[example]] +name = "ingestion_examples" +path = "ingestion_examples.rs" + +[[example]] +name = "ingestion_delete" +path = "ingestion_delete.rs" + +[[example]] +name = "ingestion_progress" +path = "ingestion_progress.rs" + +[[example]] +name = "search_vector_search" +path = "search_vector_search.rs" + +[[example]] +name = "search_text_search" +path = "search_text_search.rs" + +[[example]] +name = "search_hybrid_search" +path = "search_hybrid_search.rs" + +[[example]] +name = "search_filter_search" +path = "search_filter_search.rs" + +[[example]] +name = "search_query_plan" +path = "search_query_plan.rs" diff --git a/docs/web-tests/rs/connection.rs b/docs/web-tests/rs/connection.rs index 4f7212ef5..06fbf879f 100644 --- a/docs/web-tests/rs/connection.rs +++ b/docs/web-tests/rs/connection.rs @@ -3,48 +3,69 @@ use std::path::PathBuf; -use lancedb::connect; +use lancedb::{Connection, connect}; -// --8<-- [start:connect] -async fn connect_example(uri: &str) { - let db = connect(uri).execute().await.unwrap(); - let _ = db; +async fn connect_local() -> lancedb::Result { + // --8<-- [start:connect] + let db = connect("ex_lancedb").execute().await?; + // --8<-- [end:connect] + Ok(db) } -// --8<-- [end:connect] #[tokio::main] -async fn main() { - let temp_dir = tempfile::tempdir().unwrap(); - let uri = temp_dir.path().join("ex_lancedb"); - connect_example(uri.to_str().unwrap()).await; - - // Keep the enterprise snippet in this file, but don't run it in CI. - let _ = connect_enterprise_quickstart_config(); - let _ = connect_object_storage_config(); -} - -fn connect_enterprise_quickstart_config() -> (String, String, String, String) { - // --8<-- [start:connect_enterprise_quickstart] - let uri = "db://your-database-uri"; - let api_key = "your-api-key"; - let region = "us-east-1"; - let host_override = "https://your-enterprise-endpoint.com"; - // --8<-- [end:connect_enterprise_quickstart] - ( - uri.to_string(), - api_key.to_string(), - region.to_string(), - host_override.to_string(), +async fn main() -> Result<(), Box> { + let temp_dir = tempfile::tempdir()?; + std::env::set_current_dir(temp_dir.path())?; + let db = connect_local().await?; + db.create_empty_table( + "probe", + std::sync::Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new( + "id", + arrow_schema::DataType::Int64, + false, + )])), ) + .execute() + .await?; + let reopened = connect(temp_dir.path().join("ex_lancedb").to_str().unwrap()) + .execute() + .await?; + assert_eq!(reopened.table_names().execute().await?, vec!["probe"]); + + // Connects to a deployment only when asked, and only lists its tables. + if std::env::var("LANCEDB_DOCS_TEST_DEPLOYMENT").as_deref() == Ok("1") { + let db = connect_enterprise_quickstart().await?; + println!("{:?}", db.table_names().execute().await?); + } + Ok(()) } -fn connect_object_storage_config() -> &'static str { +async fn connect_enterprise_quickstart() -> Result> { + // --8<-- [start:connect_enterprise_quickstart] + let uri = std::env::var("LANCEDB_URI")?; // db://your-database + let api_key = std::env::var("LANCEDB_API_KEY")?; + let region = std::env::var("LANCEDB_REGION").unwrap_or_else(|_| "us-east-1".to_string()); + let host_override = std::env::var("LANCEDB_HOST_OVERRIDE")?; // https://your-endpoint + + let db = connect(&uri) + .api_key(&api_key) + .region(®ion) + .host_override(&host_override) + .execute() + .await?; + // --8<-- [end:connect_enterprise_quickstart] + Ok(db) +} + +// Compiled as documentation, never called by the example runner. +#[allow(dead_code)] +async fn connect_object_storage() -> lancedb::Result { // --8<-- [start:connect_object_storage] let uri = "s3://your-bucket/path"; // You can also use "gs://your-bucket/path" or "az://your-container/path". + let db = connect(uri).execute().await?; // --8<-- [end:connect_object_storage] - - uri + Ok(db) } async fn namespace_table_ops_example(uri: &str) -> lancedb::Result<()> { @@ -132,3 +153,17 @@ async fn namespace_admin_ops_example() -> lancedb::Result<()> { fn repo_root() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..") } + +// Compiled as documentation, never called by the example runner. +#[allow(dead_code)] +async fn connect_enterprise_placeholders() -> Result> { + // --8<-- [start:connect_enterprise_placeholders] + let db = connect("db://your-database") + .api_key("your-api-key") + .region("us-east-1") + .host_override("https://your-endpoint") + .execute() + .await?; + // --8<-- [end:connect_enterprise_placeholders] + Ok(db) +} diff --git a/docs/web-tests/rs/embedding.rs b/docs/web-tests/rs/embedding.rs index 33bdec997..67f6bbc80 100644 --- a/docs/web-tests/rs/embedding.rs +++ b/docs/web-tests/rs/embedding.rs @@ -17,7 +17,7 @@ use lancedb::{ #[tokio::main] async fn main() -> Result<()> { let db = connect("./mydb").execute().await?; - let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY is not set"); + let api_key = "your-openai-api-key".to_string(); let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model( api_key, "text-embedding-3-large", @@ -54,7 +54,7 @@ use std::sync::Arc; use lancedb::embeddings::openai::OpenAIEmbeddingFunction; -let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY is not set"); +let api_key = "your-openai-api-key".to_string(); let embedding = Arc::new( OpenAIEmbeddingFunction::new_with_model(api_key, "text-embedding-3-small") .expect("failed to create OpenAI embedding function"), @@ -77,7 +77,7 @@ use lancedb::{ #[tokio::main] async fn main() -> Result<()> { let db = connect("./mydb").execute().await?; - let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY is not set"); + let api_key = "your-openai-api-key".to_string(); let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model( api_key, "text-embedding-3-large", diff --git a/docs/web-tests/rs/ingestion_delete.rs b/docs/web-tests/rs/ingestion_delete.rs new file mode 100644 index 000000000..f23b04f2d --- /dev/null +++ b/docs/web-tests/rs/ingestion_delete.rs @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +// --8<-- [start:example] +use std::sync::Arc; + +use arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use futures::TryStreamExt; +use lancedb::query::{ExecutableQuery, QueryBase}; + +fn make_users_reader(rows: &[(i64, &str)]) -> Box { + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("name", DataType::Utf8, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.0))), + Arc::new(StringArray::from_iter_values(rows.iter().map(|r| r.1))), + ], + ) + .unwrap(); + Box::new(RecordBatchIterator::new( + vec![Ok(batch)].into_iter(), + schema, + )) +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let db = lancedb::connect(directory.path().to_str().unwrap()) + .execute() + .await?; + let table = db + .create_table( + "users", + make_users_reader(&[(1, "Alice"), (2, "Bob"), (3, "Charlie")]), + ) + .execute() + .await?; + let predicate = "id = 3"; + let selected: Vec = table + .query() + .only_if(predicate) + .execute() + .await? + .try_collect() + .await?; + println!("Selected: {selected:?}"); + table.delete(predicate).await?; + let rows: Vec = table.query().execute().await?.try_collect().await?; + println!("Remaining: {rows:?}"); + assert_eq!(table.count_rows(None).await?, 2); + assert_eq!( + table + .count_rows(Some("id = 1 AND name = 'Alice'".into())) + .await?, + 1 + ); + assert_eq!( + table + .count_rows(Some("id = 2 AND name = 'Bob'".into())) + .await?, + 1 + ); + assert_eq!(table.count_rows(Some("id = 3".into())).await?, 0); + Ok(()) +} +// --8<-- [end:example] diff --git a/docs/web-tests/rs/ingestion_examples.rs b/docs/web-tests/rs/ingestion_examples.rs new file mode 100644 index 000000000..84748c1b9 --- /dev/null +++ b/docs/web-tests/rs/ingestion_examples.rs @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +// --8<-- [start:write_workflow] +use std::sync::Arc; + +use arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use futures::TryStreamExt; +use lancedb::query::{ExecutableQuery, QueryBase}; + +fn make_users_reader(rows: &[(i64, &str, i64)]) -> Box { + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("name", DataType::Utf8, false), + Field::new("login_count", DataType::Int64, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.0))), + Arc::new(StringArray::from_iter_values(rows.iter().map(|r| r.1))), + Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.2))), + ], + ) + .unwrap(); + Box::new(RecordBatchIterator::new( + vec![Ok(batch)].into_iter(), + schema, + )) +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let db = lancedb::connect(directory.path().to_str().unwrap()) + .execute() + .await?; + let table = db + .create_table( + "users", + make_users_reader(&[(1, "Alice", 10), (2, "Bob", 20)]), + ) + .execute() + .await?; + table + .add(make_users_reader(&[(3, "Charlie", 5)])) + .execute() + .await?; + table + .update() + .only_if("id = 2") + .column("name", "'Bobby'") + .execute() + .await?; + table + .update() + .only_if("id = 2") + .column("login_count", "login_count + 1") + .execute() + .await?; + let mut merge = table.merge_insert(&["id"]); + merge + .when_matched_update_all(None) + .when_not_matched_insert_all(); + merge + .execute(make_users_reader(&[(3, "Charlie", 6), (4, "Dana", 1)])) + .await?; + let selected: Vec = table + .query() + .only_if("id = 4") + .execute() + .await? + .try_collect() + .await?; + println!("Rows selected for deletion: {selected:?}"); + table.delete("id = 4").await?; + let rows: Vec = table.query().execute().await?.try_collect().await?; + println!("Remaining rows: {rows:?}"); + assert_eq!(table.count_rows(None).await?, 3); + assert_eq!( + table + .count_rows(Some( + "id = 2 AND name = 'Bobby' AND login_count = 21".into() + )) + .await?, + 1 + ); + assert_eq!( + table + .count_rows(Some("id = 3 AND login_count = 6".into())) + .await?, + 1 + ); + Ok(()) +} +// --8<-- [end:write_workflow] diff --git a/docs/web-tests/rs/ingestion_progress.rs b/docs/web-tests/rs/ingestion_progress.rs new file mode 100644 index 000000000..afeb4c2ad --- /dev/null +++ b/docs/web-tests/rs/ingestion_progress.rs @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +// --8<-- [start:example] +use std::sync::Arc; + +use arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use futures::TryStreamExt; +use lancedb::query::ExecutableQuery; + +fn make_users_reader(rows: &[(i64, &str)]) -> Box { + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("name", DataType::Utf8, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.0))), + Arc::new(StringArray::from_iter_values(rows.iter().map(|r| r.1))), + ], + ) + .unwrap(); + Box::new(RecordBatchIterator::new( + vec![Ok(batch)].into_iter(), + schema, + )) +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let db = lancedb::connect(directory.path().to_str().unwrap()) + .execute() + .await?; + let table = db + .create_table("users", make_users_reader(&[(1, "Alice")])) + .execute() + .await?; + table + .add(make_users_reader(&[(2, "Bob"), (3, "Charlie")])) + .progress(|progress| println!("Rows written: {}", progress.output_rows())) + .execute() + .await?; + let rows: Vec = table.query().execute().await?.try_collect().await?; + println!("After insertion: {rows:?}"); + assert_eq!(table.count_rows(None).await?, 3); + assert_eq!( + table + .count_rows(Some("id = 1 AND name = 'Alice'".into())) + .await?, + 1 + ); + assert_eq!( + table + .count_rows(Some("id = 2 AND name = 'Bob'".into())) + .await?, + 1 + ); + assert_eq!(table.count_rows(Some("id = 3".into())).await?, 1); + Ok(()) +} +// --8<-- [end:example] diff --git a/docs/web-tests/rs/quickstart.rs b/docs/web-tests/rs/quickstart.rs index fd26b012c..f6ea7e0f9 100644 --- a/docs/web-tests/rs/quickstart.rs +++ b/docs/web-tests/rs/quickstart.rs @@ -1,23 +1,20 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -use std::sync::Arc; - -use arrow_array::types::Float32Type; -use arrow_array::{ - FixedSizeListArray, Int8Array, LargeStringArray, RecordBatch, RecordBatchIterator, StructArray, -}; -use arrow_schema::{DataType, Field, FieldRef, Schema}; -use lancedb::arrow::IntoPolars; -use lancedb::database::CreateTableMode; -use lancedb::query::{ExecutableQuery, QueryBase, Select}; use lancedb::{connect, table::NewColumnTransform}; -use polars::prelude::DataFrame; -use serde::{Deserialize, Serialize}; // --8<-- [start:quickstart_define_struct] +use std::sync::Arc; + +use lancedb::arrow::arrow_array::types::Float32Type; +use lancedb::arrow::arrow_array::{ + FixedSizeListArray, Int8Array, LargeStringArray, RecordBatch, RecordBatchIterator, + RecordBatchReader, StructArray, +}; +use lancedb::arrow::arrow_schema::{DataType, Field, FieldRef, Fields, Schema}; + // Define structs representing the data schema -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone)] struct Stats { strength: i8, magic: i8, @@ -25,7 +22,7 @@ struct Stats { wisdom: i8, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone)] struct Character { id: String, name: String, @@ -43,7 +40,7 @@ fn characters_schema() -> Arc { Field::new("description", DataType::LargeUtf8, false), Field::new( "stats", - DataType::Struct(arrow_schema::Fields::from(vec![ + DataType::Struct(Fields::from(vec![ Arc::new(Field::new("strength", DataType::Int8, false)), Arc::new(Field::new("magic", DataType::Int8, false)), Arc::new(Field::new("leadership", DataType::Int8, false)), @@ -58,11 +55,11 @@ fn characters_schema() -> Arc { ), ])) } -// --8<-- [end:quickstart_define_struct] -type BatchIter = Box; - -fn characters_to_reader(schema: Arc, rows: &[Character]) -> BatchIter { +fn characters_to_reader( + schema: Arc, + rows: &[Character], +) -> Box { let ids = LargeStringArray::from_iter_values(rows.iter().map(|row| row.id.as_str())); let names = LargeStringArray::from_iter_values(rows.iter().map(|row| row.name.as_str())); let roles = LargeStringArray::from_iter_values(rows.iter().map(|row| row.role.as_str())); @@ -114,6 +111,7 @@ fn characters_to_reader(schema: Arc, rows: &[Character]) -> BatchIter { schema, )) } +// --8<-- [end:quickstart_define_struct] #[tokio::main] async fn main() { @@ -166,6 +164,8 @@ async fn main() { // --8<-- [end:quickstart_data] // --8<-- [start:quickstart_create_table] + use lancedb::database::CreateTableMode; + let schema = characters_schema(); let table = db .create_table("characters", characters_to_reader(schema.clone(), &data)) @@ -175,22 +175,37 @@ async fn main() { .unwrap(); // --8<-- [end:quickstart_create_table] assert_eq!(table.count_rows(None).await.unwrap(), 3); - db.drop_table("characters", &[]).await.unwrap(); - // --8<-- [start:quickstart_create_table_no_overwrite] - let table = db - .create_table("characters", characters_to_reader(schema.clone(), &data)) + // --8<-- [start:quickstart_add_data] + let more_data = vec![Character { + id: "4".to_string(), + name: "Morgana".to_string(), + role: "Sorceress".to_string(), + description: "Powerful sorceress of Avalon.".to_string(), + stats: Stats { + strength: 2, + magic: 5, + leadership: 4, + wisdom: 4, + }, + vector: [0.3, 0.9, 0.6, 0.8], + }]; + table + .add(characters_to_reader(schema.clone(), &more_data)) .execute() .await .unwrap(); - // --8<-- [end:quickstart_create_table_no_overwrite] - assert_eq!(table.count_rows(None).await.unwrap(), 3); + // --8<-- [end:quickstart_add_data] + assert_eq!(table.count_rows(None).await.unwrap(), 4); // --8<-- [start:quickstart_vector_search_1] + use lancedb::arrow::IntoPolars; + use lancedb::query::{ExecutableQuery, QueryBase, Select}; + // Search for examples similar to a "wise magical advisor" let query_vector = [0.2, 0.8, 0.4, 0.9]; - let result: DataFrame = table + let result = table .query() .nearest_to(&query_vector) .unwrap() @@ -211,9 +226,10 @@ async fn main() { // --8<-- [end:quickstart_vector_search_1] let name_col = result.column("name").unwrap().str().unwrap(); assert_eq!(name_col.get(0).unwrap(), "Merlin"); + assert_eq!(name_col.get(1).unwrap(), "Morgana"); // --8<-- [start:quickstart_curate_with_metadata] - let curated: DataFrame = table + let curated = table .query() .nearest_to(&query_vector) .unwrap() @@ -235,9 +251,10 @@ async fn main() { // --8<-- [end:quickstart_curate_with_metadata] let curated_name_col = curated.column("name").unwrap().str().unwrap(); assert_eq!(curated_name_col.get(0).unwrap(), "Merlin"); + assert_eq!(curated_name_col.get(1).unwrap(), "Morgana"); // --8<-- [start:quickstart_output_array] - let result: DataFrame = table + let result = table .query() .nearest_to(&query_vector) .unwrap() @@ -274,7 +291,7 @@ async fn main() { // --8<-- [end:quickstart_add_feature] // --8<-- [start:quickstart_query_feature] - let features: DataFrame = table + let features = table .query() .select(Select::Columns(vec![ "name".to_string(), @@ -289,19 +306,23 @@ async fn main() { .unwrap(); println!("{features:?}"); // --8<-- [end:quickstart_query_feature] - assert!(features.column("power_score").is_ok()); + assert_eq!(features.column("power_score").unwrap().len(), 4); + + // What the quickstart's download step leaves in an application's directory. + let app = tempfile::tempdir().unwrap(); + std::fs::create_dir(app.path().join("data")).unwrap(); + std::fs::copy( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../web/static/assets/images/quickstart/sir-lancelot.jpg"), + app.path().join("data/sir-lancelot.jpg"), + ) + .unwrap(); + std::env::set_current_dir(app.path()).unwrap(); // --8<-- [start:quickstart_multimodal_bytes] - use std::sync::Arc; + use lancedb::arrow::arrow_array::BinaryArray; - use arrow_array::{ - BinaryArray, FixedSizeListArray, LargeStringArray, RecordBatch, RecordBatchIterator, - }; - use arrow_schema::{DataType, Field, Schema}; - - let image_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) - .join("../../web/static/assets/images/quickstart/sir-lancelot.jpg"); - let image_bytes = std::fs::read(image_path).unwrap(); + let image_bytes = std::fs::read("data/sir-lancelot.jpg").unwrap(); let image_schema = Arc::new(Schema::new(vec![ Field::new("id", DataType::LargeUtf8, false), @@ -333,9 +354,10 @@ async fn main() { ], ) .unwrap(); - let image_reader: Box = Box::new( - RecordBatchIterator::new(vec![Ok(image_batch)].into_iter(), image_schema), - ); + let image_reader: Box = Box::new(RecordBatchIterator::new( + vec![Ok(image_batch)].into_iter(), + image_schema, + )); let multimodal_table = db .create_table("character_images", image_reader) .mode(CreateTableMode::Overwrite) diff --git a/docs/web-tests/rs/search_filter_search.rs b/docs/web-tests/rs/search_filter_search.rs new file mode 100644 index 000000000..0417b5b15 --- /dev/null +++ b/docs/web-tests/rs/search_filter_search.rs @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +// --8<-- [start:example_body] +use arrow_array::types::Float32Type; +use arrow_array::{FixedSizeListArray, Int64Array, RecordBatch, RecordBatchIterator, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use futures::TryStreamExt; +use lancedb::query::{ExecutableQuery, QueryBase, Select}; +use std::sync::Arc; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let db = lancedb::connect(directory.path().to_str().unwrap()) + .execute() + .await?; + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("text", DataType::Utf8, false), + Field::new("category", DataType::Utf8, false), + Field::new( + "vector", + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2), + true, + ), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from(vec![1, 2, 3])), + Arc::new(StringArray::from(vec![ + "A puppy playing in the park", + "A dog resting at home", + "A guide to growing tomatoes", + ])), + Arc::new(StringArray::from(vec!["animals", "animals", "gardening"])), + Arc::new( + FixedSizeListArray::from_iter_primitive::( + [[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]] + .into_iter() + .map(|v| Some(v.into_iter().map(Some).collect::>())), + 2, + ), + ), + ], + )?; + let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema); + let table = db + .create_table( + "documents", + Box::new(reader) as Box, + ) + .execute() + .await?; + let query = table + .vector_search(&[0.0_f32, 1.0])? + .only_if("category = 'animals'") + .select(Select::columns(&["id", "text", "category", "_distance"])) + .limit(2); + let rows: Vec = query.execute().await?.try_collect().await?; + println!("{rows:?}"); + // --8<-- [end:example_body] + let ids: Vec = rows + .iter() + .flat_map(|b| { + b.column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(); + assert_eq!( + ids.into_iter().collect::>(), + std::collections::BTreeSet::from([1, 2]) + ); + assert!(rows.iter().all(|b| { + b.column_by_name("category") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .iter() + .all(|v| v == Some("animals")) + })); + // --8<-- [start:example_tail] + Ok(()) +} +// --8<-- [end:example_tail] diff --git a/docs/web-tests/rs/search_hybrid_search.rs b/docs/web-tests/rs/search_hybrid_search.rs new file mode 100644 index 000000000..6a57c259c --- /dev/null +++ b/docs/web-tests/rs/search_hybrid_search.rs @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +// --8<-- [start:example_body] +use arrow_array::types::Float32Type; +use arrow_array::{FixedSizeListArray, Int64Array, RecordBatch, RecordBatchIterator, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use futures::TryStreamExt; +use lance_index::scalar::FullTextSearchQuery; +use lancedb::index::{Index, scalar::FtsIndexBuilder}; +use lancedb::query::{ExecutableQuery, QueryBase}; +use std::sync::Arc; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let db = lancedb::connect(directory.path().to_str().unwrap()) + .execute() + .await?; + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("text", DataType::Utf8, false), + Field::new("category", DataType::Utf8, false), + Field::new( + "vector", + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2), + true, + ), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from(vec![1, 2, 3])), + Arc::new(StringArray::from(vec![ + "A puppy playing in the park", + "A dog resting at home", + "A guide to growing tomatoes", + ])), + Arc::new(StringArray::from(vec!["animals", "animals", "gardening"])), + Arc::new( + FixedSizeListArray::from_iter_primitive::( + [[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]] + .into_iter() + .map(|v| Some(v.into_iter().map(Some).collect::>())), + 2, + ), + ), + ], + )?; + let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema); + let table = db + .create_table( + "documents", + Box::new(reader) as Box, + ) + .execute() + .await?; + table + .create_index(&["text"], Index::FTS(FtsIndexBuilder::default())) + .execute() + .await?; + let query = table + .vector_search(&[1.0_f32, 0.0])? + .full_text_search(FullTextSearchQuery::new("puppy".into()).with_column("text".into())?) + .rerank(Arc::new(lancedb::rerankers::rrf::RRFReranker::default())) + .limit(2); + let rows: Vec = query.execute().await?.try_collect().await?; + println!("{rows:?}"); + // --8<-- [end:example_body] + let ids: Vec = rows + .iter() + .flat_map(|b| { + b.column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(); + assert_eq!(ids, vec![1, 2]); + let scores: Vec = rows + .iter() + .flat_map(|b| { + b.column_by_name("_relevance_score") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(); + assert!(scores[0] > scores[1]); + // --8<-- [start:example_tail] + Ok(()) +} +// --8<-- [end:example_tail] diff --git a/docs/web-tests/rs/search_query_plan.rs b/docs/web-tests/rs/search_query_plan.rs new file mode 100644 index 000000000..97290fff1 --- /dev/null +++ b/docs/web-tests/rs/search_query_plan.rs @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +// --8<-- [start:example_body] +use arrow_array::types::Float32Type; +use arrow_array::{FixedSizeListArray, Int64Array, RecordBatch, RecordBatchIterator, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use futures::TryStreamExt; +use lancedb::query::{ExecutableQuery, QueryBase, Select}; +use std::sync::Arc; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let db = lancedb::connect(directory.path().to_str().unwrap()) + .execute() + .await?; + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("text", DataType::Utf8, false), + Field::new("category", DataType::Utf8, false), + Field::new( + "vector", + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2), + true, + ), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from(vec![1, 2, 3])), + Arc::new(StringArray::from(vec![ + "A puppy playing in the park", + "A dog resting at home", + "A guide to growing tomatoes", + ])), + Arc::new(StringArray::from(vec!["animals", "animals", "gardening"])), + Arc::new( + FixedSizeListArray::from_iter_primitive::( + [[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]] + .into_iter() + .map(|v| Some(v.into_iter().map(Some).collect::>())), + 2, + ), + ), + ], + )?; + let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema); + let table = db + .create_table( + "documents", + Box::new(reader) as Box, + ) + .execute() + .await?; + let query = table + .vector_search(&[1.0_f32, 0.0])? + .select(Select::columns(&["id", "text", "_distance"])) + .limit(2); + println!("Plan: {}", query.explain_plan(false).await?); + println!("Measured execution: {}", query.analyze_plan().await?); + let rows: Vec = query.execute().await?.try_collect().await?; + println!("{rows:?}"); + // --8<-- [end:example_body] + let ids: Vec = rows + .iter() + .flat_map(|b| { + b.column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(); + assert_eq!(ids, vec![1, 2]); + // --8<-- [start:example_tail] + Ok(()) +} +// --8<-- [end:example_tail] diff --git a/docs/web-tests/rs/search_text_search.rs b/docs/web-tests/rs/search_text_search.rs new file mode 100644 index 000000000..ebb4b0292 --- /dev/null +++ b/docs/web-tests/rs/search_text_search.rs @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +// --8<-- [start:example_body] +use arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use futures::TryStreamExt; +use lance_index::scalar::FullTextSearchQuery; +use lancedb::index::{Index, scalar::FtsIndexBuilder}; +use lancedb::query::{ExecutableQuery, QueryBase, Select}; +use std::sync::Arc; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let db = lancedb::connect(directory.path().to_str().unwrap()) + .execute() + .await?; + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("text", DataType::Utf8, false), + Field::new("category", DataType::Utf8, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from(vec![1, 2, 3])), + Arc::new(StringArray::from(vec![ + "A puppy playing in the park", + "A dog resting at home", + "A guide to growing tomatoes", + ])), + Arc::new(StringArray::from(vec!["animals", "animals", "gardening"])), + ], + )?; + let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema); + let table = db + .create_table( + "documents", + Box::new(reader) as Box, + ) + .execute() + .await?; + table + .create_index(&["text"], Index::FTS(FtsIndexBuilder::default())) + .execute() + .await?; + let query = table + .query() + .full_text_search(FullTextSearchQuery::new("puppy".into()).with_column("text".into())?) + .select(Select::columns(&["id", "text", "_score"])) + .limit(2); + let rows: Vec = query.execute().await?.try_collect().await?; + println!("{rows:?}"); + // --8<-- [end:example_body] + let ids: Vec = rows + .iter() + .flat_map(|b| { + b.column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(); + assert_eq!(ids, vec![1]); + let scores: Vec = rows + .iter() + .flat_map(|b| { + b.column_by_name("_score") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(); + assert!(scores[0] > 0.0); + // --8<-- [start:example_tail] + Ok(()) +} +// --8<-- [end:example_tail] diff --git a/docs/web-tests/rs/search_vector_search.rs b/docs/web-tests/rs/search_vector_search.rs new file mode 100644 index 000000000..f2807ebef --- /dev/null +++ b/docs/web-tests/rs/search_vector_search.rs @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +// --8<-- [start:example_body] +use arrow_array::types::Float32Type; +use arrow_array::{FixedSizeListArray, Int64Array, RecordBatch, RecordBatchIterator, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use futures::TryStreamExt; +use lancedb::query::{ExecutableQuery, QueryBase, Select}; +use std::sync::Arc; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let db = lancedb::connect(directory.path().to_str().unwrap()) + .execute() + .await?; + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("text", DataType::Utf8, false), + Field::new("category", DataType::Utf8, false), + Field::new( + "vector", + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2), + true, + ), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from(vec![1, 2, 3])), + Arc::new(StringArray::from(vec![ + "A puppy playing in the park", + "A dog resting at home", + "A guide to growing tomatoes", + ])), + Arc::new(StringArray::from(vec!["animals", "animals", "gardening"])), + Arc::new( + FixedSizeListArray::from_iter_primitive::( + [[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]] + .into_iter() + .map(|v| Some(v.into_iter().map(Some).collect::>())), + 2, + ), + ), + ], + )?; + let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema); + let table = db + .create_table( + "documents", + Box::new(reader) as Box, + ) + .execute() + .await?; + let query = table + .vector_search(&[1.0_f32, 0.0])? + .distance_type(lancedb::DistanceType::L2) + .select(Select::columns(&["id", "text", "_distance"])) + .limit(2); + let rows: Vec = query.execute().await?.try_collect().await?; + println!("{rows:?}"); + // --8<-- [end:example_body] + let ids: Vec = rows + .iter() + .flat_map(|b| { + b.column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(); + assert_eq!(ids, vec![1, 2]); + let scores: Vec = rows + .iter() + .flat_map(|b| { + b.column_by_name("_distance") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(); + assert_eq!(scores[0], 0.0); + // --8<-- [start:example_tail] + Ok(()) +} +// --8<-- [end:example_tail] diff --git a/docs/web-tests/ts/connection.test.ts b/docs/web-tests/ts/connection.test.ts index f522654ec..891501d4c 100644 --- a/docs/web-tests/ts/connection.test.ts +++ b/docs/web-tests/ts/connection.test.ts @@ -1,65 +1,89 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -import { expect, jest, test } from "@jest/globals"; -import * as path from "node:path"; -import { withTempDirectory } from "./util.ts"; -// --8<-- [start:connect] +import { afterEach, describe, expect, jest, test } from "@jest/globals"; import * as lancedb from "@lancedb/lancedb"; - -async function connectExample(uri: string) { - const db = await lancedb.connect(uri); - return db; -} -// --8<-- [end:connect] +import { withTempDirectory } from "./util.ts"; test("connect to a local database", async () => { await withTempDirectory(async (tempDir) => { - const uri = path.join(tempDir, "ex_lancedb"); - const db = await connectExample(uri); - expect(db).toBeDefined(); + const cwd = process.cwd(); + process.chdir(tempDir); + try { + // --8<-- [start:connect] + const db = await lancedb.connect("ex_lancedb"); + // --8<-- [end:connect] + await db.createTable("probe", [{ id: 1 }]); + expect((await db.listTables()).tables).toEqual(["probe"]); + } finally { + process.chdir(cwd); + } + const reopened = await lancedb.connect(`${tempDir}/ex_lancedb`); + expect((await reopened.listTables()).tables).toEqual(["probe"]); }); }); async function connectEnterpriseQuickstart() { // --8<-- [start:connect_enterprise_quickstart] - const uri = "db://your-database-uri"; - const apiKey = "your-api-key"; - const region = "us-east-1"; - const hostOverride = "https://your-enterprise-endpoint.com"; + const setting = (name: string) => { + const value = process.env[name]; + if (!value) throw new Error(`Set ${name}`); + return value; + }; - const db = await lancedb.connect(uri, { - apiKey, - region, - hostOverride, + const db = await lancedb.connect(setting("LANCEDB_URI"), { + apiKey: setting("LANCEDB_API_KEY"), + region: process.env.LANCEDB_REGION ?? "us-east-1", + hostOverride: setting("LANCEDB_HOST_OVERRIDE"), }); // --8<-- [end:connect_enterprise_quickstart] return db; } -test("enterprise quickstart connect uses placeholder config", async () => { - const mockDb = { __mock: true } as unknown as Awaited< - ReturnType - >; - const spy = jest.spyOn(lancedb, "connect").mockResolvedValue(mockDb); - - const db = await connectEnterpriseQuickstart(); - expect(db).toBe(mockDb); - expect(spy).toHaveBeenCalledWith("db://your-database-uri", { - apiKey: "your-api-key", - region: "us-east-1", - hostOverride: "https://your-enterprise-endpoint.com", +describe("enterprise quickstart connect", () => { + const saved = { ...process.env }; + afterEach(() => { + process.env = { ...saved }; + }); + + test("reads the connection from the environment", async () => { + const mockDb = { __mock: true } as unknown as Awaited< + ReturnType + >; + const spy = jest.spyOn(lancedb, "connect").mockResolvedValue(mockDb); + process.env.LANCEDB_URI = "db://your-database"; + process.env.LANCEDB_API_KEY = "your-api-key"; + delete process.env.LANCEDB_REGION; + process.env.LANCEDB_HOST_OVERRIDE = "https://your-endpoint"; + + const db = await connectEnterpriseQuickstart(); + expect(db).toBe(mockDb); + expect(spy).toHaveBeenCalledWith("db://your-database", { + apiKey: "your-api-key", + region: "us-east-1", + hostOverride: "https://your-endpoint", + }); + spy.mockRestore(); + }); + + test("requires an endpoint", async () => { + process.env.LANCEDB_URI = "db://your-database"; + process.env.LANCEDB_API_KEY = "your-api-key"; + delete process.env.LANCEDB_HOST_OVERRIDE; + + await expect(connectEnterpriseQuickstart()).rejects.toThrow( + "Set LANCEDB_HOST_OVERRIDE", + ); }); - spy.mockRestore(); }); -// --8<-- [start:connect_object_storage] async function connectObjectStorageExample() { + // --8<-- [start:connect_object_storage] const uri = "s3://your-bucket/path"; // You can also use "gs://your-bucket/path" or "az://your-container/path". const db = await lancedb.connect(uri); + // --8<-- [end:connect_object_storage] return db; } -// --8<-- [end:connect_object_storage] async function namespaceTableOpsExample() { // --8<-- [start:namespace_table_ops] @@ -119,3 +143,25 @@ void [ namespaceTableOpsExample, namespaceAdminOpsExample, ]; + +test("enterprise placeholders use an explicit endpoint", async () => { + const mockDb = {} as Awaited>; + const spy = jest.spyOn(lancedb, "connect").mockResolvedValue(mockDb); + try { + // --8<-- [start:connect_enterprise_placeholders] + const db = await lancedb.connect("db://your-database", { + apiKey: "your-api-key", + region: "us-east-1", + hostOverride: "https://your-endpoint", + }); + // --8<-- [end:connect_enterprise_placeholders] + expect(db).toBe(mockDb); + expect(spy).toHaveBeenCalledWith("db://your-database", { + apiKey: "your-api-key", + region: "us-east-1", + hostOverride: "https://your-endpoint", + }); + } finally { + spy.mockRestore(); + } +}); diff --git a/docs/web-tests/ts/ingestion_examples.test.ts b/docs/web-tests/ts/ingestion_examples.test.ts new file mode 100644 index 000000000..4c6b88412 --- /dev/null +++ b/docs/web-tests/ts/ingestion_examples.test.ts @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors +import { expect, test } from "@jest/globals"; + +test("complete ingestion workflow", async () => { + // --8<-- [start:write_workflow_body] + const lancedb = await import("@lancedb/lancedb"); + const { mkdtemp, rm } = await import("node:fs/promises"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + + const uri = await mkdtemp(join(tmpdir(), "lancedb-ingestion-")); + try { + const db = await lancedb.connect(uri); + const table = await db.createTable("users", [ + { id: 1, name: "Alice", login_count: 10 }, + { id: 2, name: "Bob", login_count: 20 }, + ]); + await table.add([{ id: 3, name: "Charlie", login_count: 5 }]); + await table.update({ where: "id = 2", values: { name: "Bobby" } }); + await table.update({ + where: "id = 2", + valuesSql: { login_count: "login_count + 1" }, + }); + await table + .mergeInsert("id") + .whenMatchedUpdateAll() + .whenNotMatchedInsertAll() + .execute([ + { id: 3, name: "Charlie", login_count: 6 }, + { id: 4, name: "Dana", login_count: 1 }, + ]); + console.log( + "Rows selected for deletion:", + await table.query().where("id = 4").toArray(), + ); + await table.delete("id = 4"); + const rows = (await table.query().toArray()).sort((a, b) => a.id - b.id); + console.log(rows); + // --8<-- [end:write_workflow_body] + expect(rows.map((row) => [row.id, row.name, row.login_count])).toEqual([ + [1, "Alice", 10], + [2, "Bobby", 21], + [3, "Charlie", 6], + ]); + // --8<-- [start:write_workflow_tail] + } finally { + await rm(uri, { recursive: true, force: true }); + } + // --8<-- [end:write_workflow_tail] +}); + +test("complete delete_rows", async () => { + // --8<-- [start:delete_rows_body] + const lancedb = await import("@lancedb/lancedb"); + const { mkdtemp, rm } = await import("node:fs/promises"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const uri = await mkdtemp(join(tmpdir(), "lancedb-ingestion-")); + try { + const db = await lancedb.connect(uri); + const table = await db.createTable("users", [ + { id: 1, name: "Alice" }, + { id: 2, name: "Bob" }, + { id: 3, name: "Charlie" }, + ]); + const predicate = "id = 3"; + console.log("Selected:", await table.query().where(predicate).toArray()); + await table.delete(predicate); + const rows = (await table.query().toArray()).sort((a, b) => a.id - b.id); + console.log("Remaining:", rows); + // --8<-- [end:delete_rows_body] + expect(rows.map((row) => [row.id, row.name])).toEqual([ + [1, "Alice"], + [2, "Bob"], + ]); + // --8<-- [start:delete_rows_tail] + } finally { + await rm(uri, { recursive: true, force: true }); + } + // --8<-- [end:delete_rows_tail] +}); + +test("complete add_progress", async () => { + // --8<-- [start:add_progress_body] + const lancedb = await import("@lancedb/lancedb"); + const { mkdtemp, rm } = await import("node:fs/promises"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const uri = await mkdtemp(join(tmpdir(), "lancedb-ingestion-")); + try { + const db = await lancedb.connect(uri); + const table = await db.createTable("users", [{ id: 1, name: "Alice" }]); + await table.add( + [ + { id: 2, name: "Bob" }, + { id: 3, name: "Charlie" }, + ], + { + progress: (progress) => + console.log(`Rows written: ${progress.outputRows}`), + }, + ); + const rows = (await table.query().toArray()).sort((a, b) => a.id - b.id); + console.log("After insertion:", rows); + // --8<-- [end:add_progress_body] + expect(rows.map((row) => [row.id, row.name])).toEqual([ + [1, "Alice"], + [2, "Bob"], + [3, "Charlie"], + ]); + // --8<-- [start:add_progress_tail] + } finally { + await rm(uri, { recursive: true, force: true }); + } + // --8<-- [end:add_progress_tail] +}); diff --git a/docs/web-tests/ts/quickstart.test.ts b/docs/web-tests/ts/quickstart.test.ts index 55ae9d56c..e32a14f86 100644 --- a/docs/web-tests/ts/quickstart.test.ts +++ b/docs/web-tests/ts/quickstart.test.ts @@ -1,176 +1,263 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -import { expect, test } from "@jest/globals"; +import { describe, expect, test } from "@jest/globals"; +import { randomUUID } from "node:crypto"; +import * as fs from "node:fs"; +import * as path from "node:path"; import * as lancedb from "@lancedb/lancedb"; import { withTempDirectory } from "./util.ts"; -test("quickstart example (async)", async () => { +// What the quickstart's download step leaves in an application's directory. +const IMAGE = path.join( + __dirname, + "../../web/static/assets/images/quickstart/sir-lancelot.jpg", +); +const deployment = ["LANCEDB_URI", "LANCEDB_HOST_OVERRIDE", "LANCEDB_API_KEY"]; +// The connection settings alone never send these tests to a deployment. +const optIn = "LANCEDB_DOCS_TEST_DEPLOYMENT"; +const connections = + process.env[optIn] === "1" ? ["local", "enterprise"] : ["local"]; + +async function tableNames(db: lancedb.Connection) { + const names: string[] = []; + let pageToken: string | undefined; + do { + const page = await db.listTables({ pageToken }); + names.push(...page.tables); + pageToken = page.pageToken; + } while (pageToken); + return names; +} + +// A deployment connection on which a test creates only tables of its own: +// each name gets a suffix unique to the test and is claimed only while no +// table has it; the test drops what it claimed, whether or not it passed. +async function withOwnedTables( + db: lancedb.Connection, + fn: (db: lancedb.Connection) => Promise, +) { + const suffix = randomUUID().replaceAll("-", "").slice(0, 12); + const claimed: string[] = []; + const createTable = async ( + name: string, + data: Parameters[1], + options: Partial = {}, + ) => { + const owned = `${name}_${suffix}`; + let mode = options.mode; + if (!claimed.includes(owned)) { + if ((await tableNames(db)).includes(owned)) { + throw new Error(`${owned} already exists and is not this test's`); + } + claimed.push(owned); + mode = "create"; + } + return db.createTable(owned, data, { ...options, mode }); + }; + try { + await fn( + new Proxy(db, { + get: (target, property, receiver) => + property === "createTable" + ? createTable + : Reflect.get(target, property, receiver), + }), + ); + } finally { + const existing = await tableNames(db); + for (const name of claimed) { + if (existing.includes(name)) { + await db.dropTable(name); + } + } + } +} + +async function withDatabase( + connection: string, + fn: (db: lancedb.Connection) => Promise, +) { + if (connection === "enterprise") { + const missing = deployment.filter((name) => !process.env[name]); + if (missing.length) { + throw new Error(`${optIn}=1 needs ${missing.join(", ")}`); + } + const db = await lancedb.connect(process.env.LANCEDB_URI!, { + apiKey: process.env.LANCEDB_API_KEY!, + region: process.env.LANCEDB_REGION ?? "us-east-1", + hostOverride: process.env.LANCEDB_HOST_OVERRIDE!, + }); + await withOwnedTables(db, fn); + return; + } await withTempDirectory(async (databaseDir) => { - const db = await lancedb.connect(databaseDir); + await fn(await lancedb.connect(databaseDir)); + }); +} - // --8<-- [start:quickstart_data] - const data = [ - { - id: "1", - name: "King Arthur", - role: "King", - description: "Leader of Camelot and wielder of Excalibur.", - stats: { strength: 4, magic: 1, leadership: 5, wisdom: 4 }, - vector: [0.7, 0.1, 0.9, 0.7], - }, - { - id: "2", - name: "Merlin", - role: "Wizard", - description: "Advisor and prophet with deep magical knowledge.", - stats: { strength: 2, magic: 5, leadership: 4, wisdom: 5 }, - vector: [0.2, 0.9, 0.4, 0.9], - }, - { - id: "3", - name: "Sir Lancelot", - role: "Knight", - description: "Legendary knight known for courage and combat skill.", - stats: { strength: 5, magic: 1, leadership: 3, wisdom: 3 }, - vector: [0.9, 0.1, 0.5, 0.4], - }, - ]; - // --8<-- [end:quickstart_data] +async function createCharacters(db: lancedb.Connection) { + // --8<-- [start:quickstart_data] + const data = [ + { + id: "1", + name: "King Arthur", + role: "King", + description: "Leader of Camelot and wielder of Excalibur.", + stats: { strength: 4, magic: 1, leadership: 5, wisdom: 4 }, + vector: [0.7, 0.1, 0.9, 0.7], + }, + { + id: "2", + name: "Merlin", + role: "Wizard", + description: "Advisor and prophet with deep magical knowledge.", + stats: { strength: 2, magic: 5, leadership: 4, wisdom: 5 }, + vector: [0.2, 0.9, 0.4, 0.9], + }, + { + id: "3", + name: "Sir Lancelot", + role: "Knight", + description: "Legendary knight known for courage and combat skill.", + stats: { strength: 5, magic: 1, leadership: 3, wisdom: 3 }, + vector: [0.9, 0.1, 0.5, 0.4], + }, + ]; + // --8<-- [end:quickstart_data] - // --8<-- [start:quickstart_create_table] - let table = await db.createTable("characters", data, { mode: "overwrite" }); - // --8<-- [end:quickstart_create_table] - expect(await table.countRows()).toBe(3); - await db.dropTable("characters"); + // --8<-- [start:quickstart_create_table] + const table = await db.createTable("characters", data, { mode: "overwrite" }); + // --8<-- [end:quickstart_create_table] + expect(await table.countRows()).toBe(3); - // --8<-- [start:quickstart_create_table_no_overwrite] - table = await db.createTable("characters", data); - // --8<-- [end:quickstart_create_table_no_overwrite] - expect(await table.countRows()).toBe(3); + // --8<-- [start:quickstart_add_data] + await table.add([ + { + id: "4", + name: "Morgana", + role: "Sorceress", + description: "Powerful sorceress of Avalon.", + stats: { strength: 2, magic: 5, leadership: 4, wisdom: 4 }, + vector: [0.3, 0.9, 0.6, 0.8], + }, + ]); + // --8<-- [end:quickstart_add_data] + expect(await table.countRows()).toBe(4); + return table; +} - // --8<-- [start:quickstart_vector_search_1] - // Search for examples similar to a "wise magical advisor" - let queryVector = [0.2, 0.8, 0.4, 0.9]; +describe.each(connections)("quickstart (%s)", (connection) => { + test("create, add and search", async () => { + await withDatabase(connection, async (db) => { + const table = await createCharacters(db); - let result = await table - .search(queryVector) - .select(["name", "role", "description", "_distance"]) - .limit(2) - .toArray(); - console.table(result); - // --8<-- [end:quickstart_vector_search_1] - expect(result[0].name).toBe("Merlin"); + // --8<-- [start:quickstart_vector_search_1] + // Search for examples similar to a "wise magical advisor" + const queryVector = [0.2, 0.8, 0.4, 0.9]; - // --8<-- [start:quickstart_curate_with_metadata] - const curated = await table - .search(queryVector) - .where("stats.magic >= 4") - .select(["name", "role", "description", "_distance"]) - .limit(2) - .toArray(); - console.table(curated); - // --8<-- [end:quickstart_curate_with_metadata] - expect(curated[0].name).toBe("Merlin"); + const result = await table + .search(queryVector) + .select(["name", "role", "description", "_distance"]) + .limit(2) + .toArray(); + console.table(result); + // --8<-- [end:quickstart_vector_search_1] + expect(result.map((row) => row.name)).toEqual(["Merlin", "Morgana"]); - // --8<-- [start:quickstart_output_array] - result = await table.search(queryVector).limit(2).toArray(); - console.table(result); - // --8<-- [end:quickstart_output_array] - expect(result[0].name).toBe("Merlin"); + // --8<-- [start:quickstart_curate_with_metadata] + const curated = await table + .search(queryVector) + .where("stats.magic >= 4") + .select(["name", "role", "description", "_distance"]) + .limit(2) + .toArray(); + console.table(curated); + // --8<-- [end:quickstart_curate_with_metadata] + expect(curated.map((row) => row.name)).toEqual(["Merlin", "Morgana"]); + }); + }); - // --8<-- [start:quickstart_add_feature] - await table.addColumns([ - { - name: "power_score", - valueSql: - "cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)", - }, - ]); - // --8<-- [end:quickstart_add_feature] - const schemaWithFeature = await table.schema(); - expect(schemaWithFeature.fields.some((f) => f.name === "power_score")).toBe( - true, - ); + test("a derived feature", async () => { + await withDatabase(connection, async (db) => { + const table = await createCharacters(db); - // --8<-- [start:quickstart_query_feature] - const features = await table - .query() - .select(["name", "role", "power_score"]) - .toArray(); - console.table(features); - // --8<-- [end:quickstart_query_feature] - expect(features[0]).toHaveProperty("power_score"); - - // --8<-- [start:quickstart_multimodal_bytes] - const arrow = await import("apache-arrow"); - const path = await import("node:path"); - const { readFile } = await import("node:fs/promises"); - - const imagePath = path.join( - __dirname, - "../../web/static/assets/images/quickstart/sir-lancelot.jpg", - ); - const imageBytes = await readFile(imagePath); - const imageSchema = new arrow.Schema([ - new arrow.Field("id", new arrow.Utf8()), - new arrow.Field("description", new arrow.Utf8()), - new arrow.Field("image", new arrow.Binary()), - new arrow.Field( - "vector", - new arrow.FixedSizeList( - 4, - new arrow.Field("item", new arrow.Float32(), true), - ), - ), - ]); - const imageData = lancedb.makeArrowTable( - [ + // --8<-- [start:quickstart_add_feature] + await table.addColumns([ { - id: "lancelot", - description: "Portrait of Sir Lancelot", - image: imageBytes, - vector: [0.9, 0.1, 0.5, 0.4], + name: "power_score", + valueSql: + "cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)", }, - ], - { schema: imageSchema }, - ); - const multimodalTable = await db.createTable( - "character_images", - imageData, - { mode: "overwrite" }, - ); - // --8<-- [end:quickstart_multimodal_bytes] - expect(await multimodalTable.countRows()).toBe(1); + ]); + // --8<-- [end:quickstart_add_feature] - // --8<-- [start:quickstart_open_table] - table = await db.openTable("characters"); - // --8<-- [end:quickstart_open_table] + // --8<-- [start:quickstart_query_feature] + const features = await table + .query() + .select(["name", "role", "power_score"]) + .toArray(); + console.table(features); + // --8<-- [end:quickstart_query_feature] + expect( + features.map((row) => [row.name, row.power_score]).sort(), + ).toEqual([ + ["King Arthur", 3.5], + ["Merlin", 4], + ["Morgana", 3.75], + ["Sir Lancelot", 3], + ]); + }); + }); - // --8<-- [start:quickstart_add_data] - const moreData = [ - { - id: "4", - name: "Morgana", - role: "Sorceress", - description: "Powerful sorceress of Avalon.", - stats: { strength: 2, magic: 5, leadership: 4, wisdom: 4 }, - vector: [0.3, 0.9, 0.6, 0.8], - power_score: 3.75, - }, - ]; + test("multimodal data", async () => { + await withDatabase(connection, async (db) => { + await withTempDirectory(async (app) => { + fs.mkdirSync(path.join(app, "data")); + fs.copyFileSync(IMAGE, path.join(app, "data/sir-lancelot.jpg")); + const cwd = process.cwd(); + process.chdir(app); + try { + // --8<-- [start:quickstart_multimodal_bytes] + const arrow = await import("apache-arrow"); + const { readFile } = await import("node:fs/promises"); - // Add data to table - await table.add(moreData); - // --8<-- [end:quickstart_add_data] - expect(await table.countRows()).toBe(4); - - // --8<-- [start:quickstart_vector_search_2] - // Search for examples similar to a "powerful sorceress" - queryVector = [0.3, 0.9, 0.6, 0.8]; - - const results = await table.search(queryVector).limit(2).toArray(); - console.table(results); - // --8<-- [end:quickstart_vector_search_2] - expect(results[0].name).toBe("Morgana"); + const imageBytes = await readFile("data/sir-lancelot.jpg"); + const imageSchema = new arrow.Schema([ + new arrow.Field("id", new arrow.Utf8()), + new arrow.Field("description", new arrow.Utf8()), + new arrow.Field("image", new arrow.Binary()), + new arrow.Field( + "vector", + new arrow.FixedSizeList( + 4, + new arrow.Field("item", new arrow.Float32(), true), + ), + ), + ]); + const imageData = lancedb.makeArrowTable( + [ + { + id: "lancelot", + description: "Portrait of Sir Lancelot", + image: imageBytes, + vector: [0.9, 0.1, 0.5, 0.4], + }, + ], + { schema: imageSchema }, + ); + const multimodalTable = await db.createTable( + "character_images", + imageData, + { mode: "overwrite" }, + ); + // --8<-- [end:quickstart_multimodal_bytes] + expect(await multimodalTable.countRows()).toBe(1); + const [row] = await multimodalTable.query().select(["image"]).toArray(); + expect(Buffer.from(row.image)).toEqual(fs.readFileSync(IMAGE)); + } finally { + process.chdir(cwd); + } + }); + }); }); }); diff --git a/docs/web-tests/ts/search_examples.test.ts b/docs/web-tests/ts/search_examples.test.ts new file mode 100644 index 000000000..be4c388ee --- /dev/null +++ b/docs/web-tests/ts/search_examples.test.ts @@ -0,0 +1,222 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import { expect, test } from "@jest/globals"; + +test("complete vector_search", async () => { + // --8<-- [start:vector_search_body] + const lancedb = await import("@lancedb/lancedb"); + const { mkdtemp, rm } = await import("node:fs/promises"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const uri = await mkdtemp(join(tmpdir(), "lancedb-search-")); + try { + const db = await lancedb.connect(uri); + const table = await db.createTable("documents", [ + { + id: 1, + text: "A puppy playing in the park", + category: "animals", + vector: [1.0, 0.0], + }, + { + id: 2, + text: "A dog resting at home", + category: "animals", + vector: [0.9, 0.1], + }, + { + id: 3, + text: "A guide to growing tomatoes", + category: "gardening", + vector: [0.0, 1.0], + }, + ]); + const rows = await table + .vectorSearch([1.0, 0.0]) + .distanceType("l2") + .select(["id", "text", "_distance"]) + .limit(2) + .toArray(); + console.log(rows); + // --8<-- [end:vector_search_body] + expect(rows.map((row) => row.id)).toEqual([1, 2]); + expect(rows[0]._distance).toBe(0); + // --8<-- [start:vector_search_tail] + } finally { + await rm(uri, { recursive: true, force: true }); + } + // --8<-- [end:vector_search_tail] +}); + +test("complete text_search", async () => { + // --8<-- [start:text_search_body] + const lancedb = await import("@lancedb/lancedb"); + const { mkdtemp, rm } = await import("node:fs/promises"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const uri = await mkdtemp(join(tmpdir(), "lancedb-search-")); + try { + const db = await lancedb.connect(uri); + const table = await db.createTable("documents", [ + { id: 1, text: "A puppy playing in the park", category: "animals" }, + { id: 2, text: "A dog resting at home", category: "animals" }, + { id: 3, text: "A guide to growing tomatoes", category: "gardening" }, + ]); + await table.createIndex("text", { config: lancedb.Index.fts() }); + const rows = await table + .query() + .fullTextSearch("puppy", { columns: ["text"] }) + .select(["id", "text", "_score"]) + .limit(2) + .toArray(); + console.log(rows); + // --8<-- [end:text_search_body] + expect(rows.map((row) => row.id)).toEqual([1]); + expect(rows[0]._score).toBeGreaterThan(0); + // --8<-- [start:text_search_tail] + } finally { + await rm(uri, { recursive: true, force: true }); + } + // --8<-- [end:text_search_tail] +}); + +test("complete hybrid_search", async () => { + // --8<-- [start:hybrid_search_body] + const lancedb = await import("@lancedb/lancedb"); + const { mkdtemp, rm } = await import("node:fs/promises"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const uri = await mkdtemp(join(tmpdir(), "lancedb-search-")); + try { + const db = await lancedb.connect(uri); + const table = await db.createTable("documents", [ + { + id: 1, + text: "A puppy playing in the park", + category: "animals", + vector: [1.0, 0.0], + }, + { + id: 2, + text: "A dog resting at home", + category: "animals", + vector: [0.9, 0.1], + }, + { + id: 3, + text: "A guide to growing tomatoes", + category: "gardening", + vector: [0.0, 1.0], + }, + ]); + await table.createIndex("text", { config: lancedb.Index.fts() }); + const reranker = await lancedb.rerankers.RRFReranker.create(); + const rows = await table + .vectorSearch([1.0, 0.0]) + .fullTextSearch("puppy", { columns: ["text"] }) + .rerank(reranker) + .limit(2) + .toArray(); + console.log(rows); + // --8<-- [end:hybrid_search_body] + expect(rows.map((row) => row.id)).toEqual([1, 2]); + expect(rows[0]._relevance_score).toBeGreaterThan(rows[1]._relevance_score); + // --8<-- [start:hybrid_search_tail] + } finally { + await rm(uri, { recursive: true, force: true }); + } + // --8<-- [end:hybrid_search_tail] +}); + +test("complete filter_search", async () => { + // --8<-- [start:filter_search_body] + const lancedb = await import("@lancedb/lancedb"); + const { mkdtemp, rm } = await import("node:fs/promises"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const uri = await mkdtemp(join(tmpdir(), "lancedb-search-")); + try { + const db = await lancedb.connect(uri); + const table = await db.createTable("documents", [ + { + id: 1, + text: "A puppy playing in the park", + category: "animals", + vector: [1.0, 0.0], + }, + { + id: 2, + text: "A dog resting at home", + category: "animals", + vector: [0.9, 0.1], + }, + { + id: 3, + text: "A guide to growing tomatoes", + category: "gardening", + vector: [0.0, 1.0], + }, + ]); + const rows = await table + .vectorSearch([0.0, 1.0]) + .where("category = 'animals'") + .select(["id", "text", "category", "_distance"]) + .limit(2) + .toArray(); + console.log(rows); + // --8<-- [end:filter_search_body] + expect(rows.map((row) => row.id).sort()).toEqual([1, 2]); + expect(rows.every((row) => row.category === "animals")).toBe(true); + // --8<-- [start:filter_search_tail] + } finally { + await rm(uri, { recursive: true, force: true }); + } + // --8<-- [end:filter_search_tail] +}); + +test("complete query_plan", async () => { + // --8<-- [start:query_plan_body] + const lancedb = await import("@lancedb/lancedb"); + const { mkdtemp, rm } = await import("node:fs/promises"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const uri = await mkdtemp(join(tmpdir(), "lancedb-search-")); + try { + const db = await lancedb.connect(uri); + const table = await db.createTable("documents", [ + { + id: 1, + text: "A puppy playing in the park", + category: "animals", + vector: [1.0, 0.0], + }, + { + id: 2, + text: "A dog resting at home", + category: "animals", + vector: [0.9, 0.1], + }, + { + id: 3, + text: "A guide to growing tomatoes", + category: "gardening", + vector: [0.0, 1.0], + }, + ]); + const query = table + .vectorSearch([1.0, 0.0]) + .select(["id", "text", "_distance"]) + .limit(2); + console.log("Plan:", await query.explainPlan()); + console.log("Measured execution:", await query.analyzePlan()); + const rows = await query.toArray(); + console.log(rows); + // --8<-- [end:query_plan_body] + expect(rows.map((row) => row.id)).toEqual([1, 2]); + // --8<-- [start:query_plan_tail] + } finally { + await rm(uri, { recursive: true, force: true }); + } + // --8<-- [end:query_plan_tail] +}); diff --git a/docs/web/.mintignore b/docs/web/.mintignore new file mode 100644 index 000000000..b8da6c17a --- /dev/null +++ b/docs/web/.mintignore @@ -0,0 +1,6 @@ +# Functions are documented before their release. At release, remove these +# entries, add the pages back to the navigation in docs.json, restore the +# /feature-engineering/functions redirect to /management/function, and publish +# the drafts marked "Draft for the Functions release". +management/function.mdx +sql/ddl/function.mdx diff --git a/docs/web/api-reference/rest/index.mdx b/docs/web/api-reference/rest/index.mdx index 805f73cda..b2dd10016 100644 --- a/docs/web/api-reference/rest/index.mdx +++ b/docs/web/api-reference/rest/index.mdx @@ -35,10 +35,13 @@ To authenticate to the Enterprise REST API, you need the endpoint for your deplo ```bash export LANCEDB_API_KEY="" -export LANCEDB_URI="https://your-enterprise-endpoint.com" +export LANCEDB_HOST_OVERRIDE="https://your-enterprise-endpoint.com" export LANCEDB_DATABASE="your-database-name" ``` +`LANCEDB_HOST_OVERRIDE` is the endpoint the SDKs take as `host_override`; the +database name is the part of its `db://` URI after `db://`. + 3. If your Enterprise deployment is private, connect through the private network endpoint provided for your deployment. For example, Azure Private Link deployments commonly use a private IP or an internal DNS name as the endpoint. ### Verify authentication {#verify-authentication} @@ -46,30 +49,37 @@ export LANCEDB_DATABASE="your-database-name" 4. Check that you can reach the deployment and list tables: ```bash -curl -X GET "$LANCEDB_URI/v1/management/table" \ - -H "Content-Type: application/json" \ +curl -X GET "$LANCEDB_HOST_OVERRIDE/v1/table/" \ -H "x-api-key: $LANCEDB_API_KEY" \ -H "x-lancedb-database: $LANCEDB_DATABASE" ``` +The response lists one page of table names, with a token for the next page when +there are more. + If your deployment endpoint already includes the database host name, you can omit the `x-lancedb-database` header. -5. Create a table to confirm write access. Let's call it `words`. +5. Create a table to confirm write access. Let's call it `words`. A table is +created from an Arrow IPC stream, which this writes with PyArrow: ```bash -curl -X POST "$LANCEDB_URI/v1/tables/words" \ +python -c "import pyarrow as pa, sys; t = pa.table({'word': ['hello', 'world']}); w = pa.ipc.new_stream(sys.stdout.buffer, t.schema); w.write_table(t); w.close()" > words.arrow + +curl -X POST "$LANCEDB_HOST_OVERRIDE/v1/table/words/create" \ -H "Content-Type: application/vnd.apache.arrow.stream" \ -H "x-api-key: $LANCEDB_API_KEY" \ - -H "x-lancedb-database: $LANCEDB_DATABASE" + -H "x-lancedb-database: $LANCEDB_DATABASE" \ + --data-binary @words.arrow ``` 6. Check that the table has been created: ```bash -curl -X GET "$LANCEDB_URI/v1/management/table" \ +curl -X POST "$LANCEDB_HOST_OVERRIDE/v1/table/words/describe" \ -H "Content-Type: application/json" \ -H "x-api-key: $LANCEDB_API_KEY" \ - -H "x-lancedb-database: $LANCEDB_DATABASE" + -H "x-lancedb-database: $LANCEDB_DATABASE" \ + -d '{}' ``` That's it -- you're connected! Now, you can start adding data and querying it. diff --git a/docs/web/architecture.mdx b/docs/web/architecture.mdx index 4ee1bc0e7..997bb95b0 100644 --- a/docs/web/architecture.mdx +++ b/docs/web/architecture.mdx @@ -1,153 +1,124 @@ --- title: "Architecture" sidebarTitle: "Architecture" -description: "How LanceDB is put together, and what changes between running it embedded and running it distributed." +description: "How LanceDB Enterprise separates query serving, background work, and storage, and how it compares with embedded LanceDB OSS." icon: cubes keywords: ["architecture", "embedded", "distributed", "object storage", "compute", "deployment"] --- -LanceDB has one storage layer and two ways to put compute on top of it. Which -one you run decides where queries execute, what runs on your behalf, and what -you have to operate — and almost nothing else. The data is the same -[Lance](/basics/lance) tables either way. +LanceDB OSS is an embedded library for working with Lance tables. LanceDB +Enterprise is a multimodal lakehouse for data curation, feature engineering, +search, and training data access. Its architecture separates query serving, +read execution, caching, and background work. Both use the open +[Lance](/basics/lance) format. -## One storage layer {#storage} +Teams can curate datasets, derive features, search multimodal records, and +read the resulting tables into training frameworks. Enterprise separates the +resources that serve shared data from those that build indexes and run +background work, so teams can size them for these different workloads. -Everything durable lives in object storage: table data, manifests, indexes. No -machine owns a dataset, and no server has to be running for the data to exist. +![LanceDB OSS runs the application and embedded library in one process, using local disk or object storage. Enterprise applications connect to query serving, with separate read execution and caching, indexing and background workers, and durable object storage. Both use the Lance format, but they do not share a storage location automatically.](/static/assets/images/overview/lancedb-architecture.svg) -``` - ┌──────────────────────────────────────────┐ - │ object storage │ - │ table data · manifests · indexes │ - └──────────────────────────────────────────┘ -``` +The diagram shows the main compute and storage boundaries, not every request +path. The control plane governs the Enterprise service. -That is the whole persistence story, and it is why the two deployment shapes can -differ so much above it while staying compatible below it. A table written -embedded is opened by a deployment, and the reverse, without conversion. +## A shared format, different storage choices {#storage} -## Embedded {#embedded} +Lance stores table data, versions, and indexes. OSS can read and write tables on +local disk or in object storage. Enterprise keeps durable table data and index +artifacts in object storage, independently of the machines serving requests. +Its caches accelerate reads; they are not the durable source of truth. -``` - your process - ┌────────────────────────────────┐ - │ application │ - │ LanceDB library │ ← plans, executes, indexes, compacts - └────────────────────────────────┘ - │ - ┌─────────▼─────────┐ - │ object storage │ - └───────────────────┘ -``` +Sharing a format does not mean that an Enterprise deployment automatically +opens an existing OSS database. Plan how to move or connect your data and +configure access for your deployment. -`lancedb.connect("./data")` or `connect("s3://bucket/path")` gives you a -**library**, not a client. Your process opens the tables, plans the queries, -runs the scans, builds the indexes, and writes the new versions. There is no -server in the picture at all. +## OSS: an embedded library inside your application {#embedded} -**What follows from that.** Nothing runs unless you call it — no background -compaction, no server-side [jobs](/basics/jobs), no -[SQL endpoint](/basics/sql). Scale is your process's scale: a query gets the -cores and memory your process has. Access control is whatever your object store -enforces, since there is no other gatekeeper. +With an embedded connection, the LanceDB library plans and executes queries, +builds indexes, and updates tables inside your process. There is no separate +database server to deploy. -**What it is good at.** Development, notebooks, single-node pipelines, embedding -LanceDB inside an application, and anything where "no service to operate" is -worth more than elasticity. Comfortably handles millions of vectors on one -node -- disk-based indexes keep it performant without extra infrastructure. +This fits local retrieval applications, notebooks, development, and +single-node data pipelines. Your application controls the compute resources and +when to run [table maintenance](/maintenance/compaction). You also manage access +to the local files or object storage that hold your tables. -## Distributed {#distributed} +## Enterprise: a multimodal lakehouse for shared workloads {#distributed} -Enterprise +Applications connect to remote tables through a LanceDB SDK. The service handles +requests across a fleet, with components sized for different kinds of work: -``` - your process - ┌──────────────┐ - │ application │ - │ LanceDB SDK │ ← a thin client - └──────┬───────┘ - │ db:// (REST / Flight SQL) - ┌──────▼──────────────────────────────────────┐ - │ control plane config · identity · policy │ - ├─────────────────────────────────────────────┤ - │ data plane query serving · writes │ - │ indexers indexing · compaction │ - └──────┬──────────────────────────────────────┘ - │ - ┌──────▼────────────┐ - │ object storage │ - └───────────────────┘ -``` +- **Query serving** receives requests, resolves tables, and plans operations. + Add serving capacity as more applications and users depend on your data. +- **Read execution and caching** use separately provisioned resources. Executors + cache table data to reduce repeated reads from object storage. +- **Indexing and background workers** build indexes and perform maintenance + outside the query request path. Size this work separately from query serving + as your data grows. -`connect("db://...")` gives you a **client**. Planning and execution move to a -cluster, and the work splits across tiers that scale independently: a control -plane governing configuration, identity and policy; a data plane serving reads -and writes; and indexers building indexes and compacting outside the request -path. +The control plane manages configuration, identity, policy, and cluster +lifecycle. It governs the service rather than executing table queries. +[Authorization](/governance/authorization) lets teams control access to catalog +objects when enabled and configured by the administrator. Deployment settings govern background +[compaction](/maintenance/compaction) and +[cleanup](/maintenance/cleanup), including retention. -**What follows from that.** Compute and storage are separated, so a query fleet -scales for interactive traffic without also scaling indexing capacity, and heavy -background work does not steal resources from user queries. Because a server -exists, so do the things that need one: [SQL](/sql), -[functions](/management/function), server-side jobs, and -[authorization](/governance/authorization) per object rather than per bucket. +These boundaries support different parts of an AI data workflow. +[SQL curation](/curation) requires a deployment with SQL enabled. +[Feature engineering](/feature-engineering) can produce derived columns and +tables; computed columns and materialized views require a deployment that +enables them. [Geneva](/geneva) runs distributed Python feature backfills while +its capabilities continue to transition to LanceDB APIs. +[Training data loaders](/training/data-loading) read tables into model training +frameworks in both offerings. LanceDB provides data access for training; the +training framework runs the model. -**What it is good at.** Many concurrent queries, continuous ingestion and -indexing, multi-team access with real permissions, and workloads too large for -one machine -- customers regularly operate on billions of rows and tens of -terabytes. +Size and monitor each tier for your workload. Read the +[Enterprise architecture](/enterprise/architecture) for the component details +and request flows. -## What changes, precisely {#differences} +## Application changes for Enterprise {#differences} -| | Embedded | Distributed | -| --- | --- | --- | -| What you connect to | a library in your process | a cluster | -| Where queries plan and execute | your process | the data plane | -| Table handle | `LanceTable` | `RemoteTable` | -| Indexing and compaction | when you call it | dedicated indexers | -| [Jobs](/basics/jobs) | run in-process, no server record | real jobs: id, status, events, cancel | -| [SQL](/sql) | unavailable | Flight SQL endpoint | -| [Functions](/management/function) | unavailable | run on workers | -| Access control | object-store permissions | [per-object grants](/governance/authorization) | -| Scale bound | your process | the cluster | -| What you operate | nothing | a deployment | +Core operations remain familiar: create a table, add data, and run a search. +With OSS, the application executes those operations. With Enterprise, the SDK +sends requests to the service, and the service manages execution and configured +background maintenance. -Both expose the same table API — create, open, mutate, query — so application -code moves between them by changing the URI. What does not move is code that -depends on a row in the right-hand column. +To use Enterprise, configure the cluster endpoint, database URI and credentials, +and make your data available in the Enterprise database. Review SDK methods +that behave differently on remote tables, including result retrieval and +background jobs. [Moving to LanceDB Enterprise](/basics/sdk#moving-to-enterprise) +describes these application changes. If you plan to use materialized views, check the +[stable row ID requirements](/basics/limits#schema) before creating source tables. -## Choosing, and changing your mind {#choosing} +## Choose for the workload you need to run {#choosing} -Start embedded. It has no operational cost and the API is the same one you will -keep using. +Choose OSS when you want an embedded library and can manage its resources and +maintenance in your application. -Move when you hit something in the right-hand column — usually concurrent -serving, ingestion that outgrows one machine, or a permissions requirement. -Because the storage layer is identical, moving is a connection-string change -plus a data migration, not a rewrite. +Choose Enterprise when multiple applications or teams need a shared multimodal +lakehouse for curation, feature engineering, search, and training data access. +It also fits workloads whose query and indexing demand grow at different rates, +or that need configured service-level access controls and background operations. +You can begin with Enterprise. - -The one thing to decide early is **stable row ids**: the SDK can give them to a -table only when it creates one on a local path or an object store, and a -deployment may require them on a -[materialized view](/management/materialized-view) source. See -[Limits](/basics/limits#schema). - +[Talk to us about your workload](https://www.lancedb.com/contact) to plan capacity, +deployment options, and the path from your existing data to Enterprise. ## Where to go next {#next} + + Query serving, execution, and background work in detail. + + + Connect an application to your deployment. + What a write promises, and what a reader sees. - The open format underneath both shapes. - - - The distributed tiers in detail. - - - The bounds you can actually hit. + The open format underneath both offerings. diff --git a/docs/web/basics/agentic-skills.mdx b/docs/web/basics/agentic-skills.mdx index 319e770b4..bda6e1f65 100644 --- a/docs/web/basics/agentic-skills.mdx +++ b/docs/web/basics/agentic-skills.mdx @@ -269,10 +269,11 @@ where appropriate. Choose [LanceDB Enterprise](/enterprise) when the resulting table becomes shared production infrastructure, and the workload needs distributed capacity, -private deployment, or platform-managed operations. The underlying data format and table -API stay the same, so the pipeline does not need to be redesigned. The agent -instead connects to a remote `db://` table and lets the cluster handle -maintenance and background work. +private deployment, or platform-managed operations. The data format and the core +table API stay the same, so the pipeline's table and search code carries over: +the agent connects to a remote `db://` table, leaves maintenance to the +deployment instead of calling `optimize()`, and submits long-running work as +jobs. See [Moving to LanceDB Enterprise](/basics/sdk#moving-to-enterprise). ## What agents are good at here {#workflows} diff --git a/docs/web/basics/catalog.mdx b/docs/web/basics/catalog.mdx index 133599f16..c6950b9de 100644 --- a/docs/web/basics/catalog.mdx +++ b/docs/web/basics/catalog.mdx @@ -1,107 +1,66 @@ --- title: "Lance Catalog" sidebarTitle: "Lance Catalog" -description: "The naming layer: what objects exist, how they nest, and who may reach them." +description: "Organize, find and control access to your data in LanceDB." icon: sitemap -keywords: ["catalog", "namespace", "hierarchy", "governance", "database", "table", "view", "function"] +keywords: ["catalog", "namespace", "hierarchy", "governance", "database", "table", "view"] --- -A catalog answers one question: **what exists, and what is it called.** It holds -names, the hierarchy they sit in, and the rules about who may reach them. It -does not store rows, run queries, or build indexes -- that is the -[lakehouse](/basics/lakehouse) on top of it. +## Catalog overview {#catalog} -Keeping the two apart is what lets a Lance table be reached by something that is -not LanceDB. The catalog is a specification -- the -[Lance Namespace](/basics/lance#standard) -- so any engine that implements it -resolves the same names to the same tables. +A catalog keeps track of named data objects, such as tables and views, so +applications can discover and access them. In LanceDB, it organizes those +objects into databases and namespaces. -## The objects {#objects} +## Databases, namespaces and tables {#objects} -| Object | What it is | +A database contains namespaces that group related tables and views. Namespaces +can also contain other namespaces, allowing teams to organize data by project, +environment or purpose. + +For example, a search application can keep production and staging data in +separate namespaces within the same database: + +```mermaid +flowchart TD + database["ai_data · database"] --> prod["prod · namespace"] + database --> staging["staging · namespace"] + prod --> prod_docs["documents · table"] + staging --> staging_docs["documents · table"] +``` + + + +| Object | What it represents | | --- | --- | -| [Database](/management/database) | The root. A connection opens exactly one. | -| [Namespace](/management/namespace) | A named grouping inside a database. Can nest. | -| [Table](/management/table) | Rows and columns, versioned. What most operations act on. | -| [Materialized view](/management/materialized-view) | A table whose contents come from a query, computed and stored. | -| [View](/management/view) | A saved query. Computed when read, not stored. | -| [Function](/management/function) | Registered code, applied to produce column values. | +| [Database](/management/database) | A container for namespaces and their data objects. | +| [Namespace](/management/namespace) | A named group of tables, views or nested namespaces. | +| [Table](/management/table) | Data stored as rows and columns. | +| [View](/management/view) | A saved query, evaluated when read. Available in Enterprise. | +| [Materialized view](/management/materialized-view) | The stored results of a query, updated by a refresh. | -``` -database -└── namespace - ├── table - ├── materialized view (a table, defined by a query) - ├── view (a query, not stored) - └── function -``` +{/* +Draft for the Functions release. Add this row to the table above: +| [Function](/management/function) | Registered code used to compute column values. Available in Enterprise. | +*/} -A namespace can contain further namespaces, so the hierarchy goes as deep as -your organisation needs. A table is reached by its path through it: -`prod.events.clicks`. + -This page is about what the objects *are*. For the operations on them, see -[Management](/management). +## Organization and access control {#governance} -## Resolution {#resolution} +- **Separate projects and environments.** Keep production and staging tables + in distinct namespaces, even when they use the same table names. +- **Find shared datasets.** Give teams a consistent place to discover and + access the data they work with. +- **Control access in Enterprise.** Use permissions on databases, namespaces + and tables to manage who can access shared data and what they can do with it. -A name is resolved by walking the path, and the walk is the point: reaching a -table means traversing the namespaces above it. That is what makes the hierarchy -more than decoration -- it is the unit of both organisation and access. + -A flat set of tables needs none of this. Namespaces start earning their keep when -several teams, environments or domains share one catalog and you want -`prod.search.documents` and `staging.search.documents` to be unambiguous. +## Learn more {#next} -## Governance {#governance} - -Enterprise - -Because the catalog knows the hierarchy, it is the layer that can enforce -access. A grant names **one privilege** on **one object** for **one subject**, -and it inherits down the path: reaching a table requires `USAGE` on the -namespace containing it and on the database containing that. - -That is the practical reason to put a boundary in the hierarchy -- a namespace -per team gives you something to grant on. See -[Authorization](/governance/authorization). - -Embedded LanceDB has no such layer. Access is whatever the object store enforces -on the bucket, which has no concept of a table. - -## Views and materialized views are different objects {#views} - -The names are close enough to mislead. Both are catalog objects defined by a -query over other tables; the difference is when that query runs. - -- A **materialized view** stores its rows, and a refresh brings them up to - date. Reads are reads of stored data -- fast, and as current as the last - refresh. -- A **view** stores only the query. Every read runs it, so a view is never stale - and never faster than the query it wraps. - -Reach for a materialized view when the query is expensive and the data changes -less often than it is read. Reach for a view when correctness matters more than -latency, or when the query is cheap. - -## What the catalog does not hold {#elsewhere} - -**Indexes** belong to a table rather than standing alone. Creating one is a -performance decision, not a structural one -- see [Indexing](/indexing). - -**Jobs** are not stored data at all; they are handles to work in progress. See -[Jobs](/basics/jobs). - -**Rows** are in the [Lance](/basics/lance) files the catalog points at. The -catalog knows a table exists and where it lives; it does not know what is in it. - -## Next {#next} - - - - What operates on these objects: search, compute, maintenance. - - - Creating, listing and dropping each one. - - +- [Databases](/management/database): connect to and manage a database. +- [Namespaces](/management/namespace): create groups and address tables within them. +- [Tables](/management/table): create, inspect and manage your data. +- [Views](/management/view) and [materialized views](/management/materialized-view): work with query-defined data. +- [Authorization](/governance/authorization): configure Enterprise access permissions. diff --git a/docs/web/basics/connect.mdx b/docs/web/basics/connect.mdx new file mode 100644 index 000000000..ac7061bbc --- /dev/null +++ b/docs/web/basics/connect.mdx @@ -0,0 +1,132 @@ +--- +title: "Connect to LanceDB" +sidebarTitle: "Connect" +description: "Use the embedded LanceDB OSS library with tables in a local directory or object storage, or connect to LanceDB Enterprise." +icon: plug +keywords: ["connect", "uri", "embedded", "object storage", "s3", "enterprise", "db://"] +--- + +import { + PyConnect, + PyConnectAsync, + PyConnectEnterprisePlaceholders, + PyConnectEnterpriseQuickstart, + PyConnectObjectStorage, + PyConnectObjectStorageAsync, + RsConnect, + TsConnect, + TsConnectEnterprisePlaceholders, + TsConnectEnterpriseQuickstart, + TsConnectObjectStorage, +} from '/snippets/connection.mdx'; + +The URI you pass to `connect` decides where the work runs. The same Python, +TypeScript and Rust SDKs open all three kinds; +[OSS and Enterprise](/basics/offerings) describes how the two offerings differ. + +| URI | Where queries run | +| --- | --- | +| A local path, such as `./data` | OSS: in your process | +| `s3://`, `gs://` or `az://` | OSS: in your process, reading object storage | +| `db://` | LanceDB Enterprise: queries run on your cluster | + +## A local directory {#local} + +Embedded in your application, LanceDB reads and writes the tables in a +directory on disk. + + + + {PyConnect} + + + + {PyConnectAsync} + + + + { "import * as lancedb from \"@lancedb/lancedb\";\n\n" } + {TsConnect} + + + + { "use lancedb::connect;\n" } + { "\n" } + {RsConnect} + + + +## Object storage {#object-storage} + +With an object storage URI the library is still embedded: your process runs +every query and reads and writes the bucket directly, with the credentials from +its environment or its storage options. [Configuring storage](/deployment/storage) +covers credentials, endpoints and provider options. + + + + {PyConnectObjectStorage} + + + + {PyConnectObjectStorageAsync} + + + + { "import * as lancedb from \"@lancedb/lancedb\";\n\n" } + {TsConnectObjectStorage} + + + +In Rust, pass the same URI to `connect`. + +## LanceDB Enterprise {#enterprise} + +A `db://` URI connects to LanceDB Enterprise, where your cluster runs queries +and background work. Get the database URI, API key, region and endpoint from +your administrator, then replace the placeholders below. Use your cluster's region +in place of `us-east-1`. In application code, load credentials from environment +variables or a secrets manager. + + + + {PyConnectEnterprisePlaceholders} + + + + { "import * as lancedb from \"@lancedb/lancedb\";\n\n" } + {TsConnectEnterprisePlaceholders} + + + +In Rust, enable the crate's `remote` feature; the connection builder then takes +the same settings through `api_key`, `region` and `host_override`. To use OAuth +instead of an API key, see +[Authentication](/enterprise/authentication); for every connection option, see +[Database](/management/database#deployment). + + + + +Set these values in the shell that starts your application. Replace the URI, API +key, region and endpoint with your Enterprise connection settings. In production, a secrets +manager can supply these environment variables for you. + +```shell +export LANCEDB_URI="db://your-database" +export LANCEDB_API_KEY="your-api-key" +export LANCEDB_REGION="us-east-1" +export LANCEDB_HOST_OVERRIDE="https://your-endpoint" +``` + + + + {PyConnectEnterpriseQuickstart} + + + { "import * as lancedb from \"@lancedb/lancedb\";\n\n" } + {TsConnectEnterpriseQuickstart} + + + + diff --git a/docs/web/basics/guarantees.mdx b/docs/web/basics/guarantees.mdx index 47c7c8d7c..840ea2d50 100644 --- a/docs/web/basics/guarantees.mdx +++ b/docs/web/basics/guarantees.mdx @@ -1,134 +1,91 @@ --- title: "Guarantees" sidebarTitle: "Guarantees" -description: "What a write promises when it returns, what a reader sees, and what survives a failure." +description: "Understand table commits, concurrent writes and the boundaries of recovery." icon: shield-check -keywords: ["guarantees", "consistency", "atomicity", "durability", "concurrency", "acid"] +keywords: ["guarantees", "consistency", "atomicity", "durability", "concurrency"] --- -What the system promises, and — just as usefully — what it does not. Most -surprises in practice come from assuming one of the guarantees below is -stronger or weaker than it is. +## Atomic table writes {#writes} -## Writes {#writes} +A table transaction commits its changes atomically: readers see the committed +table version, not a partially applied transaction. For example, a committed +append adds its rows together. -**A write is atomic.** Every write commits a new table version. Either the -version exists and holds all of the change, or it does not exist and holds none -of it. There is no state in which a reader sees half an `add()`. +This boundary is one table transaction. A workflow that updates several tables +or runs a background job can contain multiple commits. Do not assume the whole +workflow is atomic. -**A write returns only after the commit.** When the call returns, the version -is durable in object storage. Nothing is buffered in your process waiting to be -flushed. + -**Every write advances the version**, including one that affects zero rows. A -delete whose predicate matches nothing still commits. Version numbers count -commits, not changes. +Schema changes are also part of table history. Changing a column's type can +rewrite data; adding a derived column can write the new column independently. -**Concurrent writes are resolved optimistically.** A transaction records the -change and the version it was based on. At commit time, a conflicting -concurrent operation is detected rather than silently overwritten, and the -commit is retried against the newer version. Operations that do not collide — -appending while another process appends — both succeed. +## Consistent reads {#reads} - -`merge_insert` reports its retry attempts in the result. A high count means -contention on the same keys, not an error. - +A read of a table uses a consistent snapshot. A consistent snapshot can still +be older than another writer's latest commit. [Consistency](/tables/consistency) +explains how OSS and Enterprise choose the version a reader sees. -## Reads {#reads} +For an experiment that must keep using the same data, select a specific retained +table version rather than relying on a connection's default freshness. -**A read sees a consistent snapshot.** A query reads one table version -throughout; it never sees a mixture of two. +## Concurrent writes {#concurrency} -**Which version, by default, is the one open at the time the table was opened.** -This is the guarantee most often assumed to be stronger than it is: +Lance uses optimistic concurrency control. Writers prepare changes, then check +compatibility with intervening commits. Compatible changes can be committed; +conflicts may be retried or returned as errors. - -A long-lived connection **does not observe other processes' writes** until it -reopens the table or its `read_consistency_interval` elapses. The default is -right for a batch job reading a stable snapshot and wrong for a service -expected to see fresh data. - +Applications must handle failed writes. Concurrent appends are compatible, but +that does not promise success through storage failures or exhausted retries. +Do not rely on a particular ordering between independent writers. -Set `read_consistency_interval` when several processes write the same table and -readers must observe those writes. The cost is a metadata check on that -interval. See [Consistency](/tables/consistency). +The [Lance transaction specification](https://lance.org/format/table/transaction/) +describes commit behavior and compatibility between operations. -**Your own writes are always visible to you.** The interval governs other -writers, not yourself. +## Storage durability {#durability} -## Durability and history {#durability} +A committed table version is stored in the table’s configured storage. LanceDB +OSS uses your local filesystem or object store; LanceDB Enterprise uses the +cluster’s configured storage. -**Old versions are retained, not overwritten.** A new version does not destroy -the one before it, which is what makes [time travel](/management/table#versioning) work -and what lets a reader finish a scan that started on an older version. +Table version history does not replace backups. Configure storage redundancy +and backups to meet your recovery requirements. -**Deletion is a mark, not an erase.** A deleted row is recorded in a deletion -file. The data is recoverable for as long as the version that contains it -exists. +## Background job completion {#jobs} - -Retention is finite. [Compaction](/maintenance/compaction) prunes versions -older than the retention window — **7 days by default** — and a tag or branch -pointing at a pruned version no longer resolves. If you rely on reading old -versions, widen the window before running it, not after. - +In LanceDB Enterprise, operations such as index builds and column refreshes +can run as background jobs. Receiving a job handle confirms submission. Wait +for the job to succeed before relying on its result. -**Dropping a table ends its history.** Versioning does not survive -`drop_table`; there is no undrop. +Some operations save results in batches. If a job fails or is cancelled, +batches already committed can remain in the table. Each operation’s guide +explains whether it can leave partial results. -## Jobs {#jobs} +## Writes after a client timeout {#timeouts} -**A refresh recomputes only what changed.** It gives values to rows that lack -them and recomputes rows whose inputs changed since they were computed; it -leaves the rest alone. Two consequences, both relied on elsewhere: +If a client times out while waiting for a write, the write may already have +committed. Retrying an append can therefore insert the same records again. -- **Retrying is safe.** Re-running a refresh fills what is still empty or stale. - It is not guaranteed to cost only the rows that did not complete: a retry can - recompute values that were already written. -- **A filled value catches up at the next refresh.** If a column derives from - `price` and a row's `price` later changes, the derived value keeps its old - value until the next refresh recomputes it. +Check whether the intended records were written before retrying. For an +operation submitted as a background job, check its status using the job handle. -**What a failure keeps depends on where the refresh ran.** `refresh_column` in -your process commits the column once, so a failure keeps none of its values. A -refresh job on a deployment commits finished batches as it goes, so some values -can remain. +## Version history and recovery {#history} -**Cancellation is a request.** `cancel()` asks a job to stop; it does not -guarantee it stops before doing more work, and cancelling a finished job does -nothing. +Retained table versions let you inspect earlier data or restore a table to a +previous version. Cleanup can remove those versions and their underlying files, +so configure retention to cover the history you need. -**A job id is opaque.** It correlates with server logs and the jobs API. -Parsing it, or storing it to resume the job later, is not supported. +Dropping a table is different from deleting records. Table version history +cannot undo `drop_table`. -See [Jobs](/basics/jobs). + -## Schema {#schema} +## Learn more {#next} -**Schema changes are versioned like data changes.** Adding, renaming, retyping -or dropping a column commits a version, so a schema change can be travelled back -through. - -**A computed column's inputs are protected.** While a declaration references a -column, that column cannot be renamed, retyped, or dropped — the declaration is -part of the schema and will not be invalidated out from under itself. - -**Most schema changes do not rewrite data.** Adding a derived column writes only -the new column. Type changes are the exception and can be substantial. - -## What is not guaranteed {#not-guaranteed} - -Worth stating plainly, because each is occasionally assumed: - -| | | -| --- | --- | -| Multi-table transactions | A commit covers one table. There is no cross-table atomic write. | -| Read-your-writes across processes | Governed by `read_consistency_interval`, not automatic. | -| Unenforced primary keys | `SET UNENFORCED PRIMARY KEY` records intent; it does not enforce uniqueness. | -| Ordering of concurrent appends | Two appends both succeed; their relative row order is not defined. | -| Index freshness | An index does not cover rows added after it was built until [optimized](/maintenance/compaction). | -| Automatic compaction | Nothing runs in the background. `optimize()` runs when you call it. | - -The last two are the most common source of "why is this query slow" — see -[Limits](/basics/limits) for the bounds that pair with them. +- [Consistency](/tables/consistency): control when reads observe updates. +- [Table versioning](/management/table#versioning): inspect and restore history. +- [Cleanup](/maintenance/cleanup): configure retention and understand protected versions. +- [Jobs](/basics/jobs): distinguish submission, completion and cancellation. +- [Column refresh](/feature-engineering/refresh-column): understand partial progress. +- [Architecture](/architecture): understand storage and background maintenance. diff --git a/docs/web/basics/jobs.mdx b/docs/web/basics/jobs.mdx index 652935ea2..0577cd120 100644 --- a/docs/web/basics/jobs.mdx +++ b/docs/web/basics/jobs.mdx @@ -1,196 +1,75 @@ --- title: "Jobs" sidebarTitle: "Job" -description: "Work that outlives the call that started it: submitting, waiting, inspecting, cancelling." +description: "Track background work from submission to completion." icon: hourglass-half -keywords: ["job", "async", "background", "events", "cancel", "refresh"] +keywords: ["job", "background", "progress", "events", "cancel"] +availability: + oss: available + enterprise: available + summary: >- + Job handles track operations in OSS and Enterprise. Enterprise also + supports listing server jobs, reopening them by ID and reading their events. --- -Some operations take longer than a request should. Building an index over a -large table, refreshing a computed column, registering a function -- these run -on the server after the call that started them has returned. +## Job handles {#convention} -Each is submitted the same way and returns the same handle. +A job handle lets you track an operation, such as an index build or a computed +column refresh. Use it to check progress, wait for completion and inspect +failures. - -A [query](/basics/query) also has an `_async` form, but its handle streams rows -rather than waiting for an outcome. Both answer the same four questions -- -which one is this, how is it going, give me the result, stop it -- and differ -only in the third. - +Methods such as Python's `create_index_async` and TypeScript's +`createIndexAsync` return job handles. In OSS, the work runs in your application +process; in Enterprise, it runs on server workers. A returned handle can +represent an operation that has already finished, so check its status before +waiting or requesting cancellation. See the operation's API reference for when +the method returns. -## The `_async` convention {#convention} +## Submit and wait for a job {#waiting} -An operation that can run in the background has two forms. The plain name blocks -until the work is done. The `_async` / `Async` name submits the work and returns -a **job** immediately. +1. **Submit the operation.** Keep the returned job handle. For an Enterprise + job, also keep its ID if another process needs to look it up later. +2. **Check progress.** Read the current status while the operation runs. +3. **Wait for completion.** Check that the job succeeded before using its + result. If it failed or was cancelled, inspect the failure details and any + results it committed before stopping. -| Blocking | Returns a job | SQL | +In Python, `job.status()` reports the current state and `job.wait()` waits for +completion, raising if the job fails or is cancelled. If `job.wait()` times out, the client stops waiting. +The timeout does not cancel the server job. + + + + +## Inspect and cancel jobs {#managing} + +| Task | What to do | +| --- | --- | +| Inspect progress | Check status and refresh the handle to retrieve details. Enterprise job events can help explain ongoing work or failures. | +| Reattach from another process | Open an existing server job with its ID, such as Python's `db.open_job(job_id)`. The server must still retain the job record. | +| Diagnose a failure | Inspect failure details and relevant events. Enterprise SQL also provides `SHOW ERRORS`. | +| Request cancellation | Call the job's cancellation method. Cancellation may take time, and work already committed is not rolled back. | + +Reopening a job retrieves the existing operation; it does not restart failed +work. Treat the ID as an opaque identifier and pass it unchanged to the jobs API. + +Event responses have [row limits](/basics/limits#jobs). An explicit limit can +request more than the default, but it does not guarantee a complete event +history. + +## OSS and Enterprise execution {#in-process} + +| | OSS | Enterprise | | --- | --- | --- | -| `create_index` | `create_index_async` | `CREATE INDEX` | -| `refresh_column` | `refresh_column_async` | `REFRESH COLUMN` | -| `create_function` | `create_function_async` | `CREATE FUNCTION` | -| `drop_table` | `drop_table_async` | `DROP TABLE` | +| Execution | In your application process | On server workers | +| Job identity | No server ID | A server ID for looking up the job | +| Event history | No server event history | Available through the jobs API | +| Reattaching after a client restart | No server job to reopen | Reopen by ID while the record is retained | - -The blocking form is not a different operation -- it submits the same job and -waits. Reach for the `_async` form when you want to do something else meanwhile, -run several at once, or survive the client going away. - +## Learn more {#next} -## Submitting and waiting {#waiting} - - -```python Python icon="python" -from lancedb.index import IvfPq - -job = table.create_index_async("vector", config=IvfPq()) - -print(job.id) # the server's id for this operation -print(job.status()) # "running" | "finished" | "failed" | "cancelled" - -job.wait() # blocks until terminal; raises if it failed -``` - -```typescript TypeScript icon="square-js" -const job = await table.createIndexAsync("vector"); - -console.log(job.id); // the server's id for this operation -console.log(await job.status()); // "running" | "finished" | "failed" | "cancelled" - -await job.wait(); // resolves when terminal; rejects if it failed -``` - - -`wait` is the one call that raises on failure. `status` is a point snapshot and -returns the failed state rather than raising, which is what you want in a -polling loop. - -In Python `wait` takes an optional timeout and returns the operation's typed -result -- a `RefreshColumnJobResult`, a `FunctionVersion`, or `None` for an -operation that has no result: - -```python -from datetime import timedelta - -result = table.refresh_column_async("embedding").wait(timeout=timedelta(minutes=30)) -``` - -`TimeoutError` means the timeout elapsed, not that the job stopped. The job is -still running; the handle just gave up watching. - - -The TypeScript `wait()` resolves to `void`. Read the outcome from `result` after -it resolves. - - -## Inspecting a job {#inspecting} - -Submitting an operation returns only an id, so the detail fields are empty until -the handle fetches the record. `refresh()` does that; `status()` does it as a -side effect. - - -```python Python icon="python" -job.refresh() - -job.state # last observed state, without another round trip -job.job_type # what kind of job, as the server names it -job.creation_ms # when it was created, epoch milliseconds -job.spec # what it was submitted with -job.result # its terminal result, once it succeeds -job.failure # why it failed, when it did -``` - -```typescript TypeScript icon="square-js" -await job.refresh(); - -job.state; // last observed state, without another round trip -job.jobType; // what kind of job, as the server names it -job.creationMs; // when it was created, epoch milliseconds -job.spec; // what it was submitted with -job.result; // its terminal result, once it succeeds -job.failure; // why it failed, when it did -``` - - -Printing the handle renders every field it knows, which is usually faster than -reading them one at a time. - -## Progress, from events {#events} - -`result` only tells you anything once the job has finished. Events are written -as it runs, and they outlive the workers that produced them -- so a job that is -still going, or one that never finished, can still account for what it did. - - -```python Python icon="python" -events = job.events(limit=5000, filter="state = 'claim_complete'") -``` - -```typescript TypeScript icon="square-js" -const events = await job.events({ limit: 5000, filter: "state = 'claim_complete'" }); -``` - -```sql SQL -SHOW ERRORS ON my_table FOR JOB 'job-id' LIMIT 100 -``` - - -A distributed job records a `claim` / `claim_complete` pair per unit of work, -each carrying `rows_processed`. Summing `rows_processed` over the completed -claims is how you get progress on a job that has no percentage. - -`filter` is a SQL-like expression over `state`, `updated_by`, `emitted_from`, -`emitted_by`, and `claim_entity`. - - -The server returns 1000 event rows by default and 10,000 at most, and -**truncates without telling you**. A job that emits an event per fragment will -silently lose its tail unless you pass `limit`. - - -Events are recorded server-side, so this raises for an operation that ran -in-process. - -## Finding and cancelling jobs {#managing} - -The connection lists and reopens jobs, so a handle lost to a restarted process -is recoverable from its id. - - -```python Python icon="python" -db.list_jobs() # every job the server knows about -job = db.open_job(job_id) # reattach to one -db.cancel_job(job_id) # ask it to stop -``` - -```typescript TypeScript icon="square-js" -await db.listJobs(); -const job = await db.openJob(jobId); -await db.cancelJob(jobId); -``` - -```sql SQL -SHOW JOBS -``` - - -Cancellation is a request, not a guarantee, and cancelling a job that already -finished does nothing. `JobNotFoundError` means the id does not name a job the -server still has. - - -A job id is opaque. It correlates with server logs and the calls above, but -parsing it or storing it to resume the job later is not supported. - - -## In-process jobs {#in-process} - -A local database has no server to run anything, so its operations run in your -process. They still return a handle, so the same code works against both -- but -one with nothing behind it: `id` is `None`, `status()` reports `"finished"`, -`wait()` returns immediately, `cancel()` does nothing, and `events()` raises. - -This is the difference to expect when moving between embedded and deployed -- -see [SDKs](/basics/sdk). +- [Build and repair indexes](/indexing/reindexing): submit index work. +- [Column refresh](/feature-engineering/refresh-column): populate derived values. +- [Background jobs](/maintenance/background-jobs): understand Enterprise maintenance. +- [SQL job commands](/sql/ddl/job): list jobs and inspect errors. +- [SDKs](/basics/sdk): find the job API for your client. diff --git a/docs/web/basics/lakehouse.mdx b/docs/web/basics/lakehouse.mdx index 7dd631ac3..04e07afaf 100644 --- a/docs/web/basics/lakehouse.mdx +++ b/docs/web/basics/lakehouse.mdx @@ -1,102 +1,75 @@ --- -title: "Lance Lakehouse" -sidebarTitle: "Lance Lakehouse" -description: "What operates on the catalog's objects: storage, search, computation, and maintenance." +title: "LanceDB Lakehouse" +sidebarTitle: "LanceDB Lakehouse" +description: "Bring multimodal data, search and feature engineering together for AI workloads." icon: layer-group -keywords: ["lakehouse", "multimodal", "search", "compute", "maintenance", "feature engineering"] +keywords: ["lakehouse", "multimodal", "search", "compute", "training", "feature engineering"] --- -The [catalog](/basics/catalog) says what exists. The **lakehouse** is everything -that acts on it: storing the rows, searching them, computing new ones, and -keeping the whole thing fast as it grows. +## Multimodal storage and processing {#multimodal} -The distinction matters because the two layers have different lifetimes. A -catalog entry is a name and a pointer. A lakehouse operation reads and writes -the [Lance](/basics/lance) files that name points at -- and those operations are -where almost all of the cost, and all of the tuning, lives. +LanceDB brings storage, search and data processing together for AI workloads. +You can store multimodal data, derive features, search your records and prepare +training datasets using tables built on the open [Lance format](/basics/lance). -## Multimodal is the point {#multimodal} +The format provides the storage foundation. The [catalog](/basics/catalog) +organizes your tables. LanceDB provides the tools to work with that data. -A lakehouse table holds images, video, audio, text, embeddings, annotations and -the features you derive from them **in one table**, side by side. +## Example: a video search workflow {#capabilities} -That is the difference from assembling a warehouse plus a vector store plus a -feature store. There is one copy of the data, one set of versions, and no -pipeline keeping three systems agreeing about the same rows. A search can filter -on a label, rank by a vector, and return the blob, without leaving the table. +Consider a team building a video search application: -| | | -| --- | --- | -| **Columns** | vectors, blobs, and columns [computed](/feature-engineering/add-computed-column) from other columns | -| **Versions** | every write commits one, so reads can [travel back](/management/table#versioning) | -| **Indexes** | vector, full-text and scalar, built over those columns | -| **Branches** | isolated lines of change you can [merge or discard](/tables/branching) | + + + -## The four capabilities {#capabilities} +1. **Ingest the clips.** Store video clips with metadata such as timestamps, + source and labels. +2. **Add features.** Generate captions and embeddings with your models, and + store them alongside each clip. +3. **Use the dataset.** Search for relevant clips using embeddings and metadata + filters, or select a dataset to read into a training framework. -Everything the lakehouse does falls into one of four groups, and the rest of -these docs is organized the same way. +```mermaid +flowchart TD + clips["Video clips and metadata"] --> table["Store in a Lance table"] + table --> features["Add captions and embeddings"] + features --> search["Search for relevant clips"] + features --> training["Select data for training"] +``` -### Ingest {#ingest} +The clips, metadata and derived features stay associated in the same table, +so retrieval and training workflows can use the same dataset. Your training +framework runs the model; LanceDB provides access to the training data. -Getting rows in, and changing the ones that are there: append, merge, update, -delete. Every write commits a version rather than mutating in place, which is -what makes the other three safe to run against a moving table. See -[Data Ingestion](/ingestion). +## Shared compute and maintenance in Enterprise {#where} -### Search {#search} +LanceDB OSS runs as an embedded library in your application. LanceDB Enterprise +provides shared infrastructure for applications and teams working with larger +multimodal workloads. -Vector, full-text and hybrid retrieval over the same rows, with filters that -push into the scan rather than running after it. The search is a lakehouse -operation, not a separate service holding a second copy of your embeddings. See -[Semantic Search](/search). + -### Compute {#compute} +- **Scale resources for different workloads.** Provision query serving and + background workers independently as search traffic and indexing demand grow. +- **Share data across teams.** Give applications access to shared tables, with + Enterprise permissions controlling who can read or change them. +- **Manage ongoing maintenance.** Configure background indexing, compaction + and cleanup as your datasets grow. -Deriving new columns and tables from what is already stored: -[computed columns](/feature-engineering/add-computed-column), -[functions](/management/function) that run Python where the data is, and -[materialized views](/management/materialized-view) that persist a query's -result. +See [Architecture](/architecture) for how Enterprise separates storage, query +serving and background work. -The reason this belongs in the lakehouse rather than in a pipeline outside it is -the format. Adding a derived column writes only that column -- the images and -videos are never rewritten -- so deriving a feature is an ordinary operation -rather than a migration. See [Feature Engineering](/feature-engineering) and -[Data Curation](/curation). +## Learn more {#next} -### Maintenance {#maintenance} +- [Data ingestion](/ingestion): load and update your data. +- [Search](/search): retrieve records with vector, full-text and hybrid search. +- [Feature engineering](/feature-engineering): derive columns and datasets. +- [Training data access](/training/data-loading): read datasets into training workflows. +- [Enterprise](/enterprise): explore deployment options for shared workloads. -The work that keeps the first three fast: building and repairing -[indexes](/indexing), and [compaction](/maintenance/compaction) to merge small -files, prune old versions and fold new rows into existing indexes. - - -None of it runs on its own. There is no background compactor -- `optimize()` -runs when you call it, and an index does not cover rows added after it was -built until you do. See [Guarantees](/basics/guarantees#not-guaranteed). - - -## Where the work happens {#where} - -The same lakehouse runs in two shapes, and which one you use decides where these -capabilities execute rather than whether you have them: - -- **Embedded** -- your process does all four itself, directly against object - storage. Nothing runs unless you call it. -- **Distributed** -- a cluster does them, with indexing and compaction on - workers separate from the ones serving queries, plus - [SQL](/sql) and server-side [jobs](/basics/jobs). - -See [Architecture](/architecture) for the comparison in full. - -## Next {#next} - - - - The objects these operations act on. - - - The open format underneath, and why column evolution is cheap. - - +{/* +Draft for the Functions release. Add this item to the list above, after +Feature engineering: +- [Functions](/management/function): run Python to compute column values. +*/} diff --git a/docs/web/basics/lance.mdx b/docs/web/basics/lance.mdx index 6552c3735..f2dd0ecfa 100644 --- a/docs/web/basics/lance.mdx +++ b/docs/web/basics/lance.mdx @@ -1,122 +1,45 @@ --- title: "Lance format" sidebarTitle: "Lance Format" -description: "Lance is the open format underneath. LanceDB is what implements a lakehouse on top of it." +description: "The open format for multimodal data behind LanceDB." icon: "/static/assets/logo/lance-logo-gray.svg" -keywords: ["lance", "format", "open standard", "lakehouse", "namespace", "catalog"] +keywords: ["lance", "format", "multimodal", "versioning", "column evolution"] --- -Two names, two things. [Lance](https://lance.org/) is an open format and -specification. LanceDB is an implementation built on it. +## Lance format overview {#standard} -The distinction matters when you are deciding what you are committing to: the -format is a standard other engines read, and your data stays readable by them -whichever LanceDB you run. +[Lance](https://lance.org/) is an open-source, columnar format designed for +multimodal AI. It defines how data is stored and organized into versioned tables. -## Lance is the open standard {#standard} + + -Lance is an open-source, columnar lakehouse format for multimodal AI. It -specifies three things: +LanceDB OSS and LanceDB Enterprise both use Lance as their storage foundation. +The format is developed as an independent open-source project, with its own +specification and tools. -| | | -| --- | --- | -| **File format** | how bytes are laid out on disk, optimized for random access without giving up scan throughput | -| **Table format** | how files compose into a versioned table -- schema evolution, deletions, indexes | -| **Namespace specification** | how tables are named, grouped, and resolved -- the catalog layer | +## Multimodal data {#storage} -Being a specification rather than a product is the point. Object storage holds -the bytes, and open engines -- PyTorch, Ray, Spark, Trino, DuckDB, Polars -- -read them without going through LanceDB at all. +A Lance table can hold images, video, audio, text, embeddings and structured +metadata together. For example, a video dataset can keep each clip alongside +its caption, labels and embeddings, so applications can search and work with +those fields together. -Lance exists because columnar formats derived from Parquet were not designed -for random access on vectors: [benchmarks](https://lancedb.com/blog/benchmarking-random-access-in-lance/) -show Lance up to 1000x faster than Parquet for that access pattern. +## Data access, versioning and column updates {#capabilities} - - The format specification: design, layout, and how it enables the multimodal lakehouse. - +- **Efficient data access.** Read individual rows for retrieval and training, + or scan columns for analytics. +- **Versioned data.** Read an earlier table version to reproduce an experiment + or inspect previous results, while that version is retained. +- **Independent column updates.** Add embeddings or other derived features + without rewriting the existing image, video or text columns. -## LanceDB embedded implements the storage half {#embedded} +## Learn more {#next} -Connecting to a directory or a bucket gives you a **storage-only lakehouse**. -Your process opens Lance tables directly and does the work itself: it plans and -executes queries, builds indexes, and writes new versions. + + -```python -db = lancedb.connect("./.lancedb") -db = lancedb.connect("s3://bucket/path") -``` - -There is no server, which is what makes it embedded -- and also what bounds it. -There is nothing to run work on your behalf, so there are no server-side -[jobs](/basics/jobs), no [SQL endpoint](/basics/sql), and no shared catalog -beyond what the storage layer itself provides. - -LanceDB OSS is Apache 2.0 licensed. LanceDB Enterprise is the commercial, -deployment-only product that implements the catalog specification below. - -## LanceDB Enterprise implements the catalog specification {#enterprise} - -Enterprise A `db://` connection talks to a deployment -that implements the **Lance REST Namespace** -- the REST server defined by the -Lance namespace specification, covering the catalog operations in the -[REST API reference](/api-reference/rest). - -```python -db = lancedb.connect("db://my-database", api_key="...") -``` - -Implementing the specification rather than inventing an interface is what makes -the catalog reachable by anything that speaks it, and what makes -[authorization](/governance/authorization), server-side jobs, and -[SQL](/basics/sql) possible: there is now a server to enforce and to run them. - -The tables are the same Lance tables either way. What changes is who resolves -their names and who does the work. - -## What the format gives you {#capabilities} - -These are properties of Lance, so they hold in both forms: - -| Capability | What it enables | -| --- | --- | -| Multimodal storage | Images, video, audio, text, embeddings, annotations and metadata in one table. | -| First-class blob API | Large binary objects in blob columns, with lazy reads and streaming byte access. | -| Fast random access and scans | Sample, shuffle, and fetch individual rows without giving up sequential throughput. | -| Zero-copy schema evolution | Add, drop, rename, or alter columns without rewriting existing data files. | -| Versioned tables | Reproduce experiments and tie artifacts to the exact table version they used. | -| Hybrid search and indexing | Vector search, full-text search, and scalar filters over the same dataset. | -| Open interoperability | Object storage underneath, open engines on top. | - -Two of them explain behaviour you will meet elsewhere in these docs. - -### Versioning is why old data lingers {#versioning} - -Every write creates a version, and old versions are not immediately destroyed -- -another reader may be part-way through one. A hundred versions are not a hundred -copies of the data, but they do carry a hundred times the metadata, which -eventually slows queries down. - -Deletion works the same way: a deleted row is marked in a deletion file rather -than erased, so the data is recoverable for as long as the version is. - -[Compaction](/maintenance/compaction) is what resolves both. - -### Column evolution is why feature engineering is cheap {#evolution} - -Adding a derived column writes only the new column. Existing data -- the -expensive part, the images and videos -- is untouched. That is what makes -[computed columns](/feature-engineering/add-computed-column) a reasonable thing to -do repeatedly on a large table rather than a full rewrite each time. - - - Lance is a separate open source project with its own documentation. - +- [Lance format specification](https://lance.org/format): how the format works. +- [Table versioning](/management/table#versioning): read and manage table versions. +- [Computed columns](/feature-engineering/add-computed-column): add derived features to your data. +- [Architecture](/architecture): how LanceDB OSS and Enterprise build on Lance. diff --git a/docs/web/basics/limits.mdx b/docs/web/basics/limits.mdx index f135de8d4..0d4f1b028 100644 --- a/docs/web/basics/limits.mdx +++ b/docs/web/basics/limits.mdx @@ -1,106 +1,71 @@ --- title: "Limits" sidebarTitle: "Limits" -description: "The bounds you can actually hit, in one place." +description: "Find fixed constraints, configurable bounds and what happens when you reach them." icon: ruler keywords: ["limits", "maximum", "quota", "constraints", "bounds"] --- -Limits worth designing around, collected so you do not have to find them by -hitting them. Each one is enforced -- where a value is refused rather than -clamped, that is stated, because the two fail very differently. + -This page covers bounds that are the same for everyone. Deployment-specific -quotas -- storage, throughput, concurrency -- come from your deployment. +This page lists limits for job events, SQL error reports, sampling and +full-text indexing. Each table identifies the affected API and what happens +when a request reaches its limit. SQL and server job limits apply to Enterprise. -## Names {#names} + -| Object | Maximum | | +[Naming Rules](/basics/naming) is the reference for name lengths, characters and +namespace depth. Client and server versions can differ in which rules they +enforce. + + + + +## Fixed API limits {#fixed-limits} + +| Operation | Scope | Limit and behavior | | --- | --- | --- | -| Database | 64 characters | | -| Namespace | 64 characters | | -| Table | 128 characters | | -| Branch | 64 characters | | +| Job event retrieval | Enterprise | At most 10,000 rows per response. Results can be truncated without a truncation indicator. | +| `SHOW ERRORS ... LIMIT` | Enterprise SQL | Accepts 1 through 100,000. Values outside the range are rejected. | +| `SHOW ERRORS` without `FOR JOB` | Enterprise SQL | Inspects up to the 32 most recent failed jobs. Use `FOR JOB` to inspect a specific job. | +| Percentage sampling | Enterprise SQL | `TABLESAMPLE` accepts percentages from 0 through 100. Values outside the range are rejected. | +| Row-count sampling | Enterprise SQL | `TABLESAMPLE (... ROWS)` requires a whole number. | -A name must not be empty, and must not contain `$` or `:` -- both are reserved -as path delimiters, so a name holding one could not be addressed unambiguously. -Length is counted in characters, not bytes. +See [SQL job commands](/sql/ddl/job#errors) and +[Sampling](/sql/dql/sampling) for syntax and restrictions on supported table types. -## Jobs {#jobs} + -| | | +Data-type and index constraints belong to the operation you choose. Check +[Data types](/sql/data-types) and [Vector indexes](/indexing/vector-index) +before choosing a schema or index. For materialized views, check +[stable row ID requirements](/feature-engineering/materialized-views#stable-row-ids) +before creating source tables. + +## Configurable limits {#not-quotas} + +| Setting | What it controls | | --- | --- | -| `job.events()` default | 1,000 rows | -| `job.events()` maximum | 10,000 rows | -| `SHOW ERRORS ... LIMIT` | between 1 and 100,000 | -| Failed jobs scanned by one `SHOW ERRORS` | 32 | +| Job event `limit` | Requests more or fewer event rows. The default is 1,000; the 10,000-row maximum still applies. | +| FTS `max_token_length` | The maximum indexed token length. The default is 40; longer tokens are excluded from the index. | +| Enterprise capacity and quotas | Storage, request rate and concurrency are governed by your deployment's resources and configuration. Confirm these with your administrator or LanceDB support. | - -`job.events()` **truncates silently**. A job that emits an event per fragment -will lose its tail unless you pass `limit` explicitly. This is the limit on this -page most likely to mislead you, because nothing in the result says it was cut. - +## Handling rejected or incomplete results {#behavior} -`SHOW ERRORS` without `FOR JOB` scans the most recent failed jobs and stops at -32 of them. Narrow with `FOR JOB` when you need a specific one rather than a -survey. +- **Rejected requests:** correct the input before retrying, such as a sampling + percentage outside the accepted range. +- **Truncated responses:** do not assume that a returned event list is the full + history. Increasing `limit` cannot exceed the maximum; use filters to narrow + the events you inspect. +- **Excluded data:** an FTS token-length setting can affect which words are + searchable. Choose it for your text and indexing requirements. -See [Jobs](/basics/jobs). + -## Schema and types {#schema} +## Learn more {#next} -| | | -| --- | --- | -| `map` with `keys_sorted = true` | rejected | -| `IVF_RQ` vector dimensions | must be divisible by 8 | -| FTS `max_token_length` | 40 by default; longer tokens are dropped | - -Through the SDK, a table gets **stable row ids** only at creation, from the -`new_table_enable_stable_row_ids` storage option on a local path or an object -store; no SDK call adds them later, so decide early. A local -[materialized view](/feature-engineering/materialized-views#stable-row-ids) does -not need them. On a deployment, whether a view's source needs them and how a -table gets them depend on the deployment; see -[Against a deployment](/feature-engineering/materialized-views#availability). - -See [Data types](/sql/data-types). - -## SQL {#sql} - -Enterprise - -| | | -| --- | --- | -| `TABLESAMPLE` percentage | 0 to 100, **refused** outside -- not clamped | -| `TABLESAMPLE (n ROWS)` | whole numbers only | -| `TABLESAMPLE BUCKET` | not supported | -| `TABLESAMPLE` on a WAL-backed table | not supported | -| `CREATE INDEX` | one column, server-chosen name | -| `REFRESH MATERIALIZED VIEW ... CASCADE` | depends on the deployment's version; see [REFRESH](/sql/dml/refresh#refresh-materialized-view) | - -A sampling percentage over 100 is an error rather than a full scan, because a -query asking for 150 percent has a mistake in it and reading it as 100 would -hide the mistake. See [Sampling](/sql/dql/sampling). - -## What is unavailable where {#availability} - -Not quotas, but the same class of surprise -- things that exist in one place and -not another. - -| | | -| --- | --- | -| SQL on a local database | unavailable; `execute_query` raises | -| SQL methods in the TypeScript SDK | none; use a Flight SQL client | -| Function methods in the TypeScript SDK | none; use Python or SQL | -| Vector search in SQL | no SQL form; use a query builder | -| Geospatial `st_*` functions | behind a build feature LanceDB does not ship | -| `tableExists` / `namespaceExists` in TypeScript | not present | -| Server-side jobs on a local database | run in-process; `events()` raises | - -See [SQL](/basics/sql) for why the first two hold. - -## Nothing here is a quota {#not-quotas} - -None of the above is a billing or capacity limit. Rows per table, table count, -storage, and request rate are bounded by your deployment rather than by the -format -- ask your deployment, not this page. +- [Naming Rules](/basics/naming): choose portable names and paths. +- [Jobs](/basics/jobs): track work and inspect events. +- [Full-text indexes](/indexing/fts-index): configure tokenization. +- [SQL](/basics/sql) and [SDKs](/basics/sdk): check interface support. +- [Enterprise support](/deployment/support): discuss deployment capacity. diff --git a/docs/web/basics/naming.mdx b/docs/web/basics/naming.mdx new file mode 100644 index 000000000..af61c2b6c --- /dev/null +++ b/docs/web/basics/naming.mdx @@ -0,0 +1,69 @@ +--- +title: "Naming rules" +sidebarTitle: "Naming Rules" +description: "Choose portable names for your data objects and namespace paths." +icon: signature +keywords: ["names", "naming", "table name", "namespace", "identifier", "limits"] +--- + +## Naming rules {#rules} + + + +Databases, namespaces, tables, views and branches all have names. Choose names +that describe their purpose, such as `documents`, `prod` or `training_2026`. +Consistent names make data easier to find and application code easier to read. + +Use the following rules for names that work across LanceDB OSS and Enterprise. + +| Object | Characters | Length | +| --- | --- | --- | +| Table, materialized view, view | ASCII letters and digits, `-`, `_`, `.` | 1 to 128 characters | +| Namespace component | ASCII letters and digits, `-`, `_`, `.` | 1 to 64 characters; at most 10 components in a path | +| Database | ASCII letters and digits, `-`, `_`, `.`, with `/` between components | 1 to 64 characters for the whole name | +| Branch | Avoid `$` and `:`; use ASCII letters, digits, `-` and `_` for portability | Keep to 1 to 64 characters | + +- Names and individual namespace or database components must not be empty. +- Table, view, namespace and database components cannot be `.` or `..`. +- Avoid `$` and `:` because they separate parts of qualified identifiers. +- A namespace path beginning with `public` cannot contain another component. + +{/* +Draft for the Functions release. Add these rows to the end of the table above: +| Secret | ASCII letters and digits, `-`, `_`, `.` | 1 to 128 characters | +| Function | ASCII letters and digits, `_` | 1 to 128 characters | +Add this item to the list above, after the rule for `.` and `..`: +- Secret names cannot be `.` or `..`. +*/} + +## Namespace paths {#paths} + +A namespace path is a list of names. `["prod", "search"]` identifies the +`search` namespace within `prod`. A table named `documents` belongs to that +namespace; its table name remains `documents`. + +Use the SDK's namespace-path argument to supply the list. A period inside an +SDK name is a character in that name, not a separator that creates another +namespace. SQL uses its own qualified-name syntax; see +[SQL namespace paths](/sql/dql/select#namespaces). + +The empty path `[]` identifies the root namespace. + +## Examples {#examples} + +| Name or path | Guidance | +| --- | --- | +| `documents`, `user-events.v2`, `train_2026` | Portable table names | +| `["prod", "search"]` | A two-component namespace path | +| `my table` | Use `my_table` instead of a space | +| `orders$2026` | Use `orders_2026`; `$` is reserved | +| `résumés` | Use an ASCII name such as `resumes` | +| `["public", "search"]` | Invalid nesting under `public`; use a different parent namespace | + + + +## Learn more {#next} + +- [Namespaces](/management/namespace): create namespaces and address tables. +- [Branches](/tables/branching): name and manage separate lines of table history. +- [Limits](/basics/limits): check other constraints before designing your schema. diff --git a/docs/web/basics/offerings.mdx b/docs/web/basics/offerings.mdx new file mode 100644 index 000000000..c66b77e78 --- /dev/null +++ b/docs/web/basics/offerings.mdx @@ -0,0 +1,70 @@ +--- +title: "LanceDB OSS or Enterprise?" +sidebarTitle: "OSS and Enterprise" +description: "Start with an embedded library, or use the LanceDB Enterprise multimodal lakehouse for shared AI workloads." +icon: compass +keywords: ["oss", "enterprise", "embedded", "deployment", "availability", "comparison"] +--- + +LanceDB gives you two ways to work with multimodal data, both built on the open +[Lance](/basics/lance) format. + +**LanceDB OSS** is an embedded library. Run it inside your application +with tables on local disk or in object storage. You control the application, +its resources, and table maintenance, without a separate database service. +[Try the quickstart](/quickstart) to create a table and run your first search. + +**LanceDB Enterprise** is a multimodal lakehouse for data curation, feature +engineering, search, and training data access. It adds query-serving capacity, +data caching, and background operations for teams working with shared +multimodal datasets. +[Connect to Enterprise](/basics/connect#enterprise) when your cluster is ready. + + + + +## Why choose Enterprise? {#why-enterprise} + +### Serve growing workloads {#serving} + +Add query-serving capacity as more applications and users depend on your data. +Enterprise spreads query traffic across a fleet and caches frequently read +table data near compute, reducing repeated reads from object storage. Query +serving and background work use separate resources, so you can size each for +your workload. See [Enterprise architecture](/enterprise/architecture). + +### Reduce operational work {#operations} + +Let the service handle routine table maintenance instead of scheduling it in +application code. Enterprise handles background [compaction](/maintenance/compaction) and can +[clean up old table versions](/maintenance/cleanup) according to your retention +policy. Configure these maintenance tasks for your Enterprise cluster so each +application does not have to schedule them. + +### Fit your deployment requirements {#deployment} + +Choose a managed deployment or bring LanceDB into your own cloud account. +Enterprise integrates with OpenTelemetry for metrics and traces, helping your +team monitor the service alongside your applications. Talk to us about your +cloud, networking, and operational requirements to plan the right deployment. +The [Enterprise overview](/enterprise) explains the available options. + +### Connect your data workflows {#workflows} + +Use the same tables for retrieval, dataset preparation, and training data access. +With its [SQL endpoint](/basics/sql) enabled, Enterprise supports analytical +[curation](/curation), including joins, grouping, and sampling. +[Feature engineering](/feature-engineering) adds derived columns and tables; +computed columns and materialized views require the corresponding features to +be enabled for your Enterprise cluster. +[Geneva](/geneva) runs Python feature backfills as distributed jobs. +[Training data loaders](/training/data-loading), available in both offerings, +read the resulting datasets for model training. + +## Plan your Enterprise deployment {#get-started} + +[Talk to us about your workload](https://www.lancedb.com/contact) to discuss your +data, query traffic, pipelines, and deployment requirements. + +For a closer look at how the service runs, read the +[Enterprise architecture](/enterprise/architecture). diff --git a/docs/web/basics/query.mdx b/docs/web/basics/query.mdx index 449c9b498..624617918 100644 --- a/docs/web/basics/query.mdx +++ b/docs/web/basics/query.mdx @@ -1,94 +1,55 @@ --- title: "Queries" sidebarTitle: "Query" -description: "Running a query, and reading its results before it finishes." +description: "Choose how to query your data and read the results." icon: magnifying-glass -keywords: ["query", "async", "streaming", "arrow", "reader", "execute_query"] +keywords: ["query", "search", "SQL", "async", "streaming", "results"] --- -A query returns rows. How you get them depends on two choices that are easy to -conflate: whether **your thread** blocks, and whether you take the results all -at once or stream them as they arrive. +## Querying tables {#queries} -## Blocking or not is a language concern {#async-client} +A query selects data from your tables. You can filter rows, choose columns, +search for similar content or combine those operations. For example, a video +application might find clips similar to a sample and return only those recorded +in the last month. -A concern of your language, not of LanceDB. The work is identical either way. +## SDK searches and SQL queries {#searches} - -```python Python icon="python" -db = lancedb.connect("db://my-db", api_key="...") # blocking -tbl = db.open_table("documents") +| Interface | When to use it | +| --- | --- | +| **SDK query builders** | Build vector, full-text and hybrid searches, apply filters and choose the fields to return. Available in OSS and Enterprise. | +| **Enterprise SQL** | Write SQL queries for filtering, aggregation, joins and other data curation tasks. Connect through Python or a Flight SQL client. | -db = await lancedb.connect_async("db://my-db", api_key="...") # coroutines -tbl = await db.open_table("documents") -``` +Both interfaces work with LanceDB tables. Choose the one that supports your +operation and fits your application. See [SQL](/basics/sql) for connection +instructions and SQL search support, and [Search](/search) for the query builders. -```typescript TypeScript icon="square-js" -// TypeScript is async throughout; there is no blocking form. -const db = await lancedb.connect("db://my-db", { apiKey: "..." }); -const tbl = await db.openTable("documents"); -``` - +## Collect or stream results {#results} -Python offers both. `AsyncConnection` exposes the same surface as coroutines, so -choosing between them is about the program you are writing -- a service wants -the async form, a script does not care. +- **Collect the results** when you want a list, Arrow table or DataFrame that + fits in memory. Use a result limit and select only the columns you need. +- **Read batches** when you want to process a large result incrementally. + Reading batches avoids loading the entire result into application memory. + It does not guarantee that the query can return its first batch immediately. -## Reading before the query finishes {#queries} +### Asynchronous clients {#async-client} -A query is not a job, but it has the same shape of choice: wait for everything, -or take a handle and start reading. +An async client lets your application do other work while a request is waiting. +That is separate from whether you collect results or read them in batches. +Python offers synchronous and asynchronous clients; TypeScript uses async APIs. +Check the SDK reference for the methods available in your client. - -```python Python icon="python" -reader = db.execute_query("SELECT id, text FROM my_docs LIMIT 1000") -table = reader.read_all() +### Query handles and background jobs {#jobs} -query = db.execute_query_async("SELECT id, text FROM my_docs") -``` +An Enterprise SQL query handle lets you inspect query execution and request +cancellation. Background operations such as index builds return a +[job](/basics/jobs) handle instead. Use the API for the returned handle to +check status and retrieve results. -```sql SQL -SELECT id, text FROM my_docs LIMIT 1000 -``` - +## Learn more {#next} -`execute_query` returns a blocking Arrow reader **once results start arriving** --- it does not wait for the whole query to finish. Results stream as record -batches, so a large result set does not have to fit in memory before you see its -first row. - -`execute_query_async` returns the query handle instead: - -```python -query = db.execute_query_async("SELECT id, text FROM my_docs") - -query.id # identifies it on the server -query.describe() # what it is doing, without consuming results -reader = query.reader() # stream the rows when you want them -query.cancel() # stop it -``` - -`describe_query(id)` reports on one from elsewhere, the way `open_job(id)` does -for a job. - - -SQL requires a deployment. See [SQL](/basics/sql#sql-is-remote). - - -Query-builder searches -- vector, full-text, hybrid -- are ordinary calls that -return results. They are fast enough not to need a handle. See -[Query](/search). - -## Searches are ordinary calls {#searches} - -Query-builder searches -- [vector](/search/vector-search), -[full-text](/search/full-text-search), [hybrid](/search/hybrid-search) -- return -results directly. They are fast enough not to need a handle, and they have no -SQL form. See [Semantic Search](/search). - -## When the work outlives the call {#jobs} - -Some operations take longer than a query should: building an index, refreshing a -computed column, registering a function. Those return a **job** rather than -rows, and a job is a different handle with a different shape. See -[Jobs](/basics/jobs). +- [Quickstart](/quickstart): run a complete create, insert and search example. +- [Search](/search): choose vector, full-text or hybrid retrieval. +- [Filtering](/search/filtering): narrow results with predicates. +- [SQL](/basics/sql): connect a client and run SQL queries. +- [SDKs](/basics/sdk): choose a language and find its API reference. diff --git a/docs/web/basics/sdk.mdx b/docs/web/basics/sdk.mdx index 3f8170e11..fc386bc7a 100644 --- a/docs/web/basics/sdk.mdx +++ b/docs/web/basics/sdk.mdx @@ -1,99 +1,117 @@ --- title: "SDKs" sidebarTitle: "SDKs" -description: "The language clients — Python, TypeScript, Rust — and what stays true across all of them." +description: "Choose a LanceDB SDK and find the API reference for your language." icon: code-compare keywords: ["python", "typescript", "rust", "java", "sdk", "rest", "api reference"] +availability: + oss: available + enterprise: available + summary: >- + Python, TypeScript and Rust support both LanceDB OSS and LanceDB Enterprise. + Java provides a client for the Enterprise REST API. --- -A language SDK is the primary way to work with LanceDB: create and manage -tables, write and query data, build indexes, run feature engineering. The same -table API works whether you're connected to an embedded database or an -Enterprise deployment -- application code moves between the two by changing -the connection URI. +Use a LanceDB SDK to create tables, add data, build indexes and run searches +from your application. Python, TypeScript and Rust work with both the embedded +OSS library and LanceDB Enterprise. -## Officially supported SDKs {#supported-sdks} +The [quickstart](/quickstart) includes installation instructions and complete +examples in all three languages. For connection settings, see +[Connect to LanceDB](/basics/connect). -Python, TypeScript and Rust are supported against both embedded and deployed -LanceDB. +## Official SDKs {#supported-sdks} -| Reference | Description | -|:--------------|-------------------| -| [Python](https://lancedb.github.io/lancedb/python/python/) | Full-featured Python client with pandas and numpy integration | -| [TypeScript](https://lancedb.github.io/lancedb/js/) | A TypeScript wrapper around the Rust library, built with `napi-rs` | -| [Rust](https://docs.rs/lancedb/latest/lancedb/index.html) | Native Rust library with persistent storage and high performance | +| SDK | Use it with | API reference | +| --- | --- | --- | +| Python | Python applications, notebooks, pandas and PyArrow | [Python reference](https://lancedb.github.io/lancedb/python/python/) | +| TypeScript | Node.js and TypeScript applications | [TypeScript reference](https://lancedb.github.io/lancedb/js/) | +| Rust | Rust applications using the native library | [Rust reference](https://docs.rs/lancedb/latest/lancedb/index.html) | +| Java | Applications using the Enterprise REST API | [Java reference](https://lancedb.github.io/lancedb/java/java/) | -Not every SDK covers the same surface, though: +Guides show the available language examples together. Use the API reference for +method signatures and options; SDKs do not expose every feature through the same +methods. In Rust, enable the `remote` feature to connect to Enterprise. -| | Python | TypeScript | -| --- | :---: | :---: | -| Create and manage catalog objects | yes | yes | -| Insert, merge, update, delete | yes | yes | -| Filter, project, aggregate, join | yes | yes | -| Vector search | yes | yes | -| Full-text search | yes | yes | -| Feature engineering, jobs | yes | partly | -| Runs against a local database | yes | yes | - -TypeScript has no methods for [functions](/management/function). Reach the -gap with SQL instead -- see [SQL](/basics/sql). +For SQL joins, aggregations and window functions, use +[SQL](/basics/sql). Python provides a SQL client, and applications in other +languages can use a Flight SQL client. TypeScript's table query API does not +provide a SQL execution method. ## REST API {#rest} -Enterprise A deployment implements the -[Lance REST Namespace](/basics/lance#enterprise), so the catalog is reachable -over HTTP by anything that speaks it -- see the -[REST API reference](/api-reference/rest). - -| Reference | Description | -|:--------------|-------------------| -| [Java](https://lancedb.github.io/lancedb/java/java/) | REST API Enterprise SDK in Java | +LanceDB Enterprise also exposes an HTTP API for catalog and table operations. +Use it when integrating a language or tool without a LanceDB SDK. The +[REST API reference](/api-reference/rest) describes its endpoints and request +formats. For SQL queries, connect through [Flight SQL](/basics/sql). ## Community SDKs {#community-sdks} -Contributed by the community rather than supported by LanceDB, so feature parity -and release cadence vary. +These clients are maintained by the community. Check each project's supported +features, installation instructions and releases before choosing one. -| Reference | Description | -|:--------------|-------------------| -| [Go](https://pkg.go.dev/github.com/lancedb/lancedb-go/pkg/lancedb) | Community-contributed Go SDK | -| [Ruby](https://github.com/scientist-labs/lancelot) | Community-contributed Ruby bindings | -| [Swift](https://github.com/RyanLisse/LanceDbSwiftKit) | Community-contributed Swift SDK | -| [R](https://github.com/CathalByrneGit/lancedb) | Community-contributed R package | -| [Flutter](https://github.com/Alexcn/flutter_lancedb) | Community-contributed Flutter bindings | - -For any other language, [SQL](/basics/sql) reaches the same tables without a -LanceDB-specific SDK at all. +| Language | Project | +| --- | --- | +| Go | [lancedb-go](https://pkg.go.dev/github.com/lancedb/lancedb-go/pkg/lancedb) | +| Ruby | [Lancelot](https://github.com/scientist-labs/lancelot) | +| Swift | [LanceDbSwiftKit](https://github.com/RyanLisse/LanceDbSwiftKit) | +| R | [lancedb](https://github.com/CathalByrneGit/lancedb) | +| Flutter | [flutter_lancedb](https://github.com/Alexcn/flutter_lancedb) | ## Python multiprocessing {#multiprocessing} -Do not use `fork` as the start method with Python's `multiprocessing` module. -Lance is multi-threaded internally, and `fork` with alive threads is unsafe -- -use `spawn` instead. See -[this discussion](https://discuss.python.org/t/concerns-regarding-deprecation-of-fork-with-alive-threads/33555) -for why. +Use `spawn` when starting Python worker processes with `multiprocessing`. +LanceDB uses threads internally, and forking a process with active threads can +cause deadlocks. Create a connection inside each worker instead of sharing an +open connection from the parent process. -## What stays the same {#invariants} +For PyTorch worker setup and connection factories, see +[PyTorch integration](/training/torch). -Whichever you pick: +## Moving to LanceDB Enterprise {#moving-to-enterprise} -- **Types are Arrow types.** A vector column is a `FixedSizeList` - everywhere, not a SQL type mapped onto storage. See - [Functions and types](/sql/functions). -- **Reads are Arrow.** Nothing is serialized into language objects on the way out - unless you convert it. -- **The catalog is one catalog.** A table created from TypeScript is visible to - Python and to SQL immediately -- there is no per-language namespace. +The core table workflow stays familiar: connect, create or open a table, write +records, and query them. Enterprise runs those operations on remote compute and +manages background maintenance. -## What to read next {#next} +- **Configure the connection.** Replace the local path with your database URI, + and supply the endpoint, region and credentials provided by your administrator. + [Connect to Enterprise](/basics/connect#enterprise) shows these settings. +- **Retrieve query results.** Python's remote tables do not support whole-table + `to_arrow()` or `to_pandas()` calls. Use a query and select the columns and + number of rows you need, for example `table.search().limit(10).to_arrow()`. +- **Handle table listings and namespaces.** Remote table listings are paginated. + Use the namespace and pagination options described in + [Namespaces](/management/namespace). +- **Use server-side jobs.** Index builds and feature refreshes can run as + [jobs](/basics/jobs). Each operation's guide explains how to submit work and + track completion. +- **Let Enterprise manage maintenance.** Use + [background maintenance](/maintenance/background-jobs) rather than calling + the embedded table's `optimize()` method. + +For SQL connection details, see [SQL](/basics/sql). For the operating model and +deployment options, see [LanceDB Enterprise](/enterprise). + +## Shared data across languages {#invariants} + +LanceDB uses Arrow schemas across its SDKs. A fixed-size vector column stores +vectors of a consistent dimension; the [data types reference](/sql/data-types) +describes the supported element types and other column types. + +Applications in different languages can work with the same tables. Connect to +the same database and namespace, use the appropriate credentials, and account +for [read consistency](/tables/consistency) when another process writes data. + +## Related guides {#next} - The same tables, reached by any Flight SQL client -- and the one way in for a language with no SDK. + Query Enterprise tables with SQL from Python or a Flight SQL client. - Give a coding agent a current reference to this same API surface. + Use the LanceDB plugin to help a coding agent work with your data. diff --git a/docs/web/basics/sql.mdx b/docs/web/basics/sql.mdx index 6b74486eb..1c4ad46cd 100644 --- a/docs/web/basics/sql.mdx +++ b/docs/web/basics/sql.mdx @@ -1,125 +1,159 @@ --- title: "SQL" sidebarTitle: "SQL" -description: "The same tables, reached with SQL — the universal access path, and where every operation is formally documented." +description: "Query LanceDB Enterprise with SQL from Python or an Arrow Flight SQL client." icon: table-list keywords: ["sql", "flight sql", "datafusion", "reference", "dialect"] +availability: + oss: unavailable + enterprise: available + summary: >- + Enterprise supports SQL through Arrow Flight SQL. Use the SDK table APIs for embedded OSS tables. See [Connection settings](#sql-is-remote) to connect. --- -Enterprise SQL runs on a deployment. A local -database has no engine to plan a statement, so `execute_query` raises there. +Use SQL in LanceDB Enterprise to filter records, join tables, calculate +aggregates, and prepare datasets. SQL queries work with the same Lance tables +you create and update through the SDKs, without exporting the data to another +system. -SQL is a second way to reach the same tables the [SDKs](/basics/sdk) do -- -not a different product, a different interface onto one engine. A table -written from Python is queried from SQL without an export step, and a table -created from SQL is visible to every SDK immediately. +You can run SQL through the Python SDK or an Arrow Flight SQL client. This +guide covers connection settings, a first query, and how SQL fits alongside +the SDK table APIs. -## Any language, via Flight SQL {#flight-sql} +## Connect a SQL client {#flight-sql} -SQL is served over an +Enterprise exposes an [Arrow Flight SQL](https://arrow.apache.org/docs/format/FlightSql.html) -endpoint, so any Flight SQL client reaches it -- which makes SQL the way in -for a language LanceDB has no SDK for. You don't need a LanceDB-specific -client at all, just a Flight SQL driver for your language. +endpoint. A compatible client sends SQL statements to this endpoint and +receives results as Arrow record batches. -The TypeScript SDK itself has no SQL method, for the same reason a Go or C# -application would reach for SQL first: connect a Flight SQL client instead of -waiting on a native binding. See [SDKs](/basics/sdk) for the officially -supported and community clients, and reach for Flight SQL whenever yours -isn't among them. +- **Python:** use `execute_query()` on an Enterprise connection, as shown below. +- **TypeScript:** use a separate Flight SQL client. The LanceDB TypeScript SDK + does not expose a SQL execution method. +- **Other languages and tools:** use a Flight SQL client or a tool with Flight + SQL integration. A LanceDB-specific SDK is not required to submit SQL. -## Every SDK operation, in SQL {#sdk-parity} +The [SQL reference](/sql#connecting) shows a query from Python. See +[SDKs](/basics/sdk) for the LanceDB language clients. -Broadly, a public SDK method has a corresponding SQL statement -- creating a -table is `CREATE TABLE`, inserting rows is `INSERT INTO`, building an index is -`CREATE INDEX`. The two describe the same operations, so whichever one your -code speaks, the other is available too. +### Connection settings {#sql-is-remote} -| | SDK | SQL | -| --- | :---: | :---: | -| Create and manage catalog objects | yes | partly | -| Insert, merge, update, delete | yes | yes | -| Filter, project, aggregate, join | yes | yes | -| Vector search | yes | version-dependent | -| Full-text search | yes | yes | -| Feature engineering, jobs | yes | yes | -| Runs against a local database | yes | **no** | +Get the following values from your Enterprise administrator: -## Vector search depends on the deployment {#no-vector-sql} +- Database URI, such as `db://your-database`. +- API key and region. +- API endpoint for `host_override`. +- Flight SQL endpoint for `sql_host_override`, including its scheme and port. -The SDK query builder provides vector search across embedded and deployed -databases. SQL support depends on the deployment version and supported query -shape. Some versions recognize `ORDER BY distance(vector, query_vector) LIMIT k` -over a table scan and plan vector search, including indexed and flat search. -Confirm support on your deployment and use SQL -[`EXPLAIN`](/sql/dql/explain) to check the chosen plan; this is not a guarantee -that every deployed release supports that form. +The Flight SQL endpoint must be enabled and reachable from your client. Pass +it explicitly as `sql_host_override` in Python, as in the example below. The +API and SQL endpoints may use different hosts or ports. -Full-text search is available through the [`fts()`](/sql/functions#fts) table -function and the [`contains_tokens`](/sql/scalar-functions#contains-tokens) -predicate. +See [Connect to Enterprise](/basics/connect#enterprise) for other connection options. -The reverse gap also exists: a [view](/management/view) is a saved SQL query, -so SQL is the only way to create one -- there's nothing for an SDK method to -call. +SQL runs in Enterprise. Connections to a local directory or an object-storage +bucket use the embedded OSS library and cannot execute SQL statements. Use +SDK table operations to query those tables. -## SQL needs a deployment {#sql-is-remote} +## Run a query in Python {#sql-from-python} -A connection to a directory or a bucket has no server to plan and execute a -statement, and asking it to raises `NotImplementedError`. This is the largest -behavioral difference between the embedded and deployed forms, and the reason -the SDKs are the portable choice: SDK code moves between the two by changing -the URI, SQL does not. +Install the SDK with `pip install lancedb`. Replace the connection placeholders +with your Enterprise settings, including the region and both endpoint URLs. +This example reads an existing table named `my_docs` with `id` and `text` +columns. Change the query to match your table. ```python -db = lancedb.connect("./.lancedb") # no SQL -db = lancedb.connect("db://my-database") # SQL -``` +import lancedb -## Running SQL from Python {#sql-from-python} +db = lancedb.connect( + "db://your-database", + api_key="your-api-key", + region="us-east-1", + host_override="https://your-api-endpoint", + sql_host_override="https://your-sql-endpoint:443", +) -```python reader = db.execute_query("SELECT id, text FROM my_docs LIMIT 10") table = reader.read_all() +print(table.to_pylist()) ``` -`execute_query` returns a blocking Arrow reader once results start arriving; it -does not wait for the whole query. `execute_query_async` returns a handle -instead -- see [Queries](/basics/query). +In application code, load credentials from environment variables or a secrets +manager. The [connection guide](/basics/connect#enterprise) shows that setup. -## Choosing between them {#choosing} +`execute_query()` returns a blocking Arrow reader when results start arriving. +The example uses `read_all()` to collect the small result in memory. For larger +results, consume the reader in batches. To submit a query and check its status +separately, use `execute_query_async()`. See [Queries](/basics/query) for query +handles, streaming, and cancellation. -Reach for **the SDKs** when you are writing data, managing the catalog, running -vector search, or building an application that should work the same embedded and -deployed. +## Choose SQL or SDK table operations {#choosing} -Reach for **SQL** when the question is analytical -- joins across tables, -aggregates, window functions -- when the thing asking is a BI tool, a -notebook, or a person who already thinks in SQL, or when it's the only client -available for your language. +Use **SQL** for joins, aggregations, window functions, and analytical queries +written in SQL. Use **SDK table operations** to build applications around table +management, ingestion, filtering, and search, including applications that run +with the embedded OSS library. -The two compose. A common shape is SDK code to build and maintain tables, and -SQL for the analysis on top. +You can combine them. For example, create and populate a table with an SDK, +then query it with SQL to prepare a training dataset. In Python, both workflows +can use the same Enterprise connection. -## The complete reference {#reference} +### Common SQL operations {#sdk-parity} -This page is the concept; [SQL Reference](/sql) is the specification. It has -one page per statement, syntax first -- and since every SDK operation has a -SQL form, those pages end up defining the precise behavior of the SDK -operations too, not just the SQL dialect. +Several tasks are available through both interfaces, but the methods and SQL +statements do not have identical coverage. SDK support also differs by +language. These SQL reference pages describe the supported statements: + +| Task | SQL reference | +| --- | --- | +| Create a table | [CREATE TABLE](/sql/ddl/table) | +| Insert or change rows | [INSERT](/sql/dml/insert), [UPDATE](/sql/dml/update), [DELETE](/sql/dml/delete) | +| Build an index | [CREATE INDEX](/sql/ddl/index) | +| Join and aggregate data | [SELECT](/sql/dql/select) | +| Save a query as a view | [CREATE VIEW](/sql/ddl/view) | + +[Computed columns](/feature-engineering/add-computed-column) store SQL +expressions whose values are populated by a refresh. {/* Draft for the Functions release: +They are separate from registered Python Functions. */} +See the computed-column and +[materialized-view](/management/materialized-view) guides for their SDK and SQL +operations and Enterprise setup requirements. + +## Vector and full-text search {#no-vector-sql} + +SQL vector search support depends on the Enterprise server version and query +form. Versions that support `ORDER BY distance(vector, query_vector) LIMIT k` +over a table scan can plan indexed or flat vector search. Check that your +server version supports this form, then use +[`EXPLAIN`](/sql/dql/explain) to inspect the query plan. + +The [SDK vector search API](/search/vector-search) is also available in +Enterprise and OSS. Use it when you prefer the SDK query builder or need to +search embedded tables. + +For full-text search in SQL, use the [`fts()`](/sql/functions#fts) table +function or the [`contains_tokens`](/sql/scalar-functions#contains-tokens) +predicate. + +## SQL syntax and API references {#reference} + +The [SQL reference](/sql) documents statement syntax, supported expressions, +and data types. For SDK methods and their arguments, use the +[Python](https://lancedb.github.io/lancedb/python/python/), +[TypeScript](https://lancedb.github.io/lancedb/js/), or +[Rust](https://docs.rs/lancedb/latest/lancedb/index.html) API reference. - DDL by object, then DML, then DQL -- the detailed technical documentation - that defines the behavior of every operation and SDK API. + Statement syntax, expressions, and data types for LanceDB SQL. -## What to read next {#next} +## Next steps {#next} - - The language clients, and what stays true across all of them. + + Read results, track long-running queries, and cancel work. - - Give a coding agent a current reference to this same API surface. + + Compare language clients and find their API references. diff --git a/docs/web/curation/aggregate.mdx b/docs/web/curation/aggregate.mdx index 91c32fdd4..b6d02d7fb 100644 --- a/docs/web/curation/aggregate.mdx +++ b/docs/web/curation/aggregate.mdx @@ -1,113 +1,144 @@ --- title: "Profiling a dataset" sidebarTitle: "Aggregate" -description: "Counting, grouping and summarizing — finding out what your data actually contains." +description: "Measure coverage, missing values and class balance before selecting a dataset." icon: chart-column keywords: ["aggregate", "group by", "class balance", "count", "profiling", "distinct"] +availability: + oss: unavailable + enterprise: available + summary: >- + Enterprise supports SQL aggregation. --- -Enterprise Aggregation is SQL, and -[SQL needs a deployment](/basics/sql#sql-is-remote). +Use SQL aggregates to summarize a dataset: count records, find missing values, +and compare groups. These summaries help you understand the data before +selecting records for search, evaluation or training. -Before curating anything, find out what you have. Most of the surprises in a -training set — a class that is 90% of the rows, a column that is half null, a -"unique" id that is not — are one `GROUP BY` away from being visible. +## Example data {#example-data} -## How big, and how complete {#size} +Connect with an [Enterprise SQL client](/basics/sql). The examples use a small +document table and a separate label table. Expand the setup below to create them. + + + +`curation_documents` has six records describing four documents. +`curation_labels` has one label for each of three documents. `row_id` identifies +a record; `doc_id` identifies the document it describes. + +Run each statement separately in a scratch database or namespace. Create these +tables once for the whole section, using fresh names if they already exist. ```sql -SELECT - count(*) AS rows, - count(embedding) AS embedded, - count(caption) AS captioned -FROM images +CREATE TABLE curation_documents ( + row_id BIGINT, doc_id BIGINT, version BIGINT, + source VARCHAR, caption VARCHAR, score DOUBLE +); +INSERT INTO curation_documents VALUES + (1, 10, 1, 'web', NULL, 0.80), + (2, 10, 2, 'web', 'A cat on a chair', 0.95), + (3, 20, 1, 'books', 'A dog in a field', 0.90), + (4, 30, 1, 'web', NULL, 0.70), + (5, 30, 1, 'web', 'A cat by a window', 0.85), + (6, 40, 1, 'books', 'A bird in a tree', 0.60); + +CREATE TABLE curation_labels (doc_id BIGINT, label VARCHAR); +INSERT INTO curation_labels VALUES + (10, 'cat'), (20, 'dog'), (30, 'cat'); ``` -`count(*)` counts rows; `count(col)` counts **non-null values**. The gap between -them is your coverage, which is the fastest way to check how far a -[column refresh](/feature-engineering/refresh-column) has got without waiting -for the job to report. +`score` is a sample quality score, not a search result score. Document 10 has +two versions. Document 30 has two records at the same version. Document 40 +has no label. These cases let you check the effect of each operation. -## Class balance {#balance} + -The single most useful curation query: +## Count records and missing values {#size} + +Use `count(*)` to count rows and `count(column)` to count non-null values. +Add `DISTINCT` to count unique values: ```sql -SELECT label, count(*) AS n -FROM images +SELECT count(*) AS records, + count(DISTINCT doc_id) AS documents, + count(caption) AS captioned, + count(*) FILTER (WHERE caption IS NULL) AS missing_caption +FROM curation_documents; +``` + +The example returns **6 records, 4 documents, 4 populated captions and 2 missing +captions**. `count(column)` excludes null values; it does not check whether a +string is empty or whether a value is useful for the task. + +## Group records by label {#balance} + +Use `GROUP BY` to calculate a summary for each label: + +```sql +SELECT label, count(*) AS documents +FROM curation_labels GROUP BY label -ORDER BY n DESC +ORDER BY documents DESC, label; ``` -If the top class is an order of magnitude larger than the rest, sampling -uniformly will reproduce that imbalance. Two ways out: draw a fixed number per -class with [windowing](/curation/windowing#per-group), or oversample the rare -ones and accept the duplication. +There are **2 cats and 1 dog** in the label table. Document 40 is absent, so this +query does not measure labeling coverage across all source documents. Use a +[left join](/curation/join#join-type) to find documents without labels. -## Distinct values {#distinct} +Decide whether the distribution matches your intended use. A uniform random +sample tends to preserve the source distribution; it does not balance classes. +To cap the number selected per label, use +[ranking within a group](/curation/windowing#per-group). + +## Count records that match a condition {#conditional} ```sql -SELECT count(DISTINCT source) FROM images +SELECT source, + count(*) AS records, + count(*) FILTER (WHERE score >= 0.9) AS high_score, + count(*) FILTER (WHERE caption IS NULL) AS missing_caption +FROM curation_documents +GROUP BY source +ORDER BY source; ``` -On a high-cardinality column this is expensive, because exact distinctness means -holding every value seen. When an estimate will do: +`FILTER` applies a condition to one aggregate. Here, `books` has 2 records, +1 high-score record and no missing captions. `web` has 4 records, 1 high-score +record and 2 missing captions. Counts are per record, so repeated document IDs +still contribute more than once. + + +## Group numeric values into ranges {#buckets} + +Use `count(DISTINCT column)` for the number of distinct non-null values. +For numeric fields, group by a range or summarize their distribution. For +example, this query counts records in score buckets of width 0.1: ```sql -SELECT approx_distinct(source) FROM images +SELECT floor(score * 10) / 10 AS score_bucket, count(*) AS records +FROM curation_documents +GROUP BY score_bucket +ORDER BY score_bucket; ``` -Cheap enough to run on a table where the exact version is not worth the wait. -The other approximate functions — `approx_median`, `approx_percentile_cont` — -are useful for the same reason when profiling a numeric column. +For time-based data, group by the time interval relevant to the task. The +[aggregation reference](/sql/dql/aggregation) and +[scalar function reference](/sql/scalar-functions) cover additional summaries, +including approximate counts and percentiles. Use an approximate aggregate when an estimate is sufficient; use an exact +count when the result must be precise. -## Conditional counts {#conditional} +## Profile a sample {#sampling} -`FILTER` narrows one aggregate without narrowing the query, which is how you get -several slices in one pass rather than one query each: +A [sample](/curation/sampling) can help you explore a large dataset before a +full scan. Report the sample size and method alongside the result. A sample can miss rare classes, and fragment sampling can reflect the order +in which records were written. -```sql -SELECT - label, - count(*) AS total, - count(*) FILTER (WHERE score > 0.9) AS confident, - count(*) FILTER (WHERE caption IS NULL) AS uncaptioned -FROM images -GROUP BY label -``` +Before accepting a curated dataset, check its required fields and key +constraints directly rather than inferring them from a small sample. -## Grouping by a bucket {#buckets} +## Continue curating {#reference} -Raw values are rarely the grouping you want. `date_bin` buckets timestamps, and -ordinary arithmetic buckets numbers: - -```sql -SELECT date_bin(INTERVAL '1 day', created_at) AS day, count(*) -FROM images GROUP BY day ORDER BY day -``` - -A histogram of a score column is the same trick: - -```sql -SELECT floor(score * 10) / 10 AS bucket, count(*) -FROM images GROUP BY bucket ORDER BY bucket -``` - -## Profiling a sample {#sampling} - -On a very large table, profile a [sample](/curation/sampling) and scale: - -```sql -SELECT label, count(*) * 100 AS estimated -FROM images TABLESAMPLE (1) -GROUP BY label -``` - -Treat the result as an estimate. A 1% sample will miss classes rarer than about -1 in 10,000 entirely, which is exactly the tail you are usually looking for — so -use this to check the shape of the head, not the existence of the tail. - -## Reference {#reference} - -The complete clause and function list is in -[Aggregation](/sql/dql/aggregation). +Use these measurements to choose a [sample](/curation/sampling), +[attach missing metadata](/curation/join), or +[select one record per document](/curation/windowing). +See [Aggregation](/sql/dql/aggregation) for complete SQL syntax. diff --git a/docs/web/curation/index.mdx b/docs/web/curation/index.mdx index 0bfc89b2d..fd8ea73aa 100644 --- a/docs/web/curation/index.mdx +++ b/docs/web/curation/index.mdx @@ -1,54 +1,125 @@ --- title: "Data curation" sidebarTitle: "Overview" -description: "Shaping a dataset before you train on it: profiling, sampling, joining, and ranking." +description: "Prepare datasets for search, evaluation and training by profiling, selecting and combining records." icon: filter keywords: ["curation", "dataset", "training set", "sampling", "deduplication", "class balance"] --- -Curation is the work between having data and having a *dataset*: finding out -what you actually have, drawing a subset you can afford to train on, attaching -labels that live elsewhere, and picking one row per thing when you have several. +Data curation turns a collection of records into a dataset suited to a task. +You might select labeled examples for training, prepare a fixed evaluation +set, or remove duplicate documents from a search corpus. -These pages are task-oriented. Each shows the operation in the context you would -reach for it, and links to the [SQL reference](/sql) for the complete syntax. +Start with the result you need. Then inspect the source data, choose which +records belong, and save the selected data or the rules used to produce it. - -Most curation is analytical -- grouping, joining, ranking -- which the query -builders do not do. It is therefore SQL, and SQL needs a deployment. See -[SQL](/basics/sql#sql-is-remote). - +## Choose a curation task {#questions} -## The four questions {#questions} - -| | | +| Task | What it helps you decide | | --- | --- | -| [What do I have?](/curation/aggregate) | counts, class balance, null coverage, distinct values | -| [What can I afford?](/curation/sampling) | a random subset, reproducible when you need it fixed | -| [What else do I know?](/curation/join) | labels, metadata and scores that live in another table | -| [Which one do I keep?](/curation/windowing) | one row per key, ranking within a group, splits | +| [Profile a dataset](/curation/aggregate) | Whether coverage, missing values and class balance suit the task. | +| [Sample records](/curation/sampling) | Which subset to inspect or use for an experiment. | +| [Join datasets](/curation/join) | How to attach labels, metadata or scores stored elsewhere. | +| [Rank and deduplicate](/curation/windowing) | Which records to keep when several describe the same item. | -## A worked shape {#shape} +The analytical examples in this section use **Enterprise SQL**. Connect with +the [SQL guide](/basics/sql). For sampling through the Python SDK in OSS or +Enterprise, see [Sample without SQL](/curation/sampling#training). -The four compose into the same pipeline most curation ends up being: +## Prepare a labeled document dataset {#shape} + +Suppose you need one labeled record per document for a training experiment. +Inspect the source records, choose the latest version of each document, then +join its label. Finally, check the class balance of the selected records. + +### Example data {#example-data} + +Connect with an [Enterprise SQL client](/basics/sql). The examples use a small +document table and a separate label table. Expand the setup below to create them. + + + +`curation_documents` has six records describing four documents. +`curation_labels` has one label for each of three documents. `row_id` identifies +a record; `doc_id` identifies the document it describes. + +Run each statement separately in a scratch database or namespace. Create these +tables once for the whole section, using fresh names if they already exist. + +```sql +CREATE TABLE curation_documents ( + row_id BIGINT, doc_id BIGINT, version BIGINT, + source VARCHAR, caption VARCHAR, score DOUBLE +); +INSERT INTO curation_documents VALUES + (1, 10, 1, 'web', NULL, 0.80), + (2, 10, 2, 'web', 'A cat on a chair', 0.95), + (3, 20, 1, 'books', 'A dog in a field', 0.90), + (4, 30, 1, 'web', NULL, 0.70), + (5, 30, 1, 'web', 'A cat by a window', 0.85), + (6, 40, 1, 'books', 'A bird in a tree', 0.60); + +CREATE TABLE curation_labels (doc_id BIGINT, label VARCHAR); +INSERT INTO curation_labels VALUES + (10, 'cat'), (20, 'dog'), (30, 'cat'); +``` + +`score` is a sample quality score, not a search result score. Document 10 has +two versions. Document 30 has two records at the same version. Document 40 +has no label. These cases let you check the effect of each operation. + + + +The following query chooses the latest record for each document, then keeps +only documents with a label: ```sql WITH ranked AS ( - SELECT d.*, c.label, - row_number() OVER (PARTITION BY d.doc_id ORDER BY d.version DESC) AS rn - FROM documents AS d - JOIN labels AS c ON c.doc_id = d.doc_id + SELECT *, row_number() OVER ( + PARTITION BY doc_id ORDER BY version DESC, row_id DESC + ) AS rn + FROM curation_documents ) -SELECT * FROM ranked WHERE rn = 1 +SELECT d.row_id, d.doc_id, d.caption, l.label +FROM ranked AS d +JOIN curation_labels AS l ON l.doc_id = d.doc_id +WHERE d.rn = 1 +ORDER BY d.doc_id; ``` -Join to bring in what you know, window to pick one row per document, filter to -keep it. Add [`TABLESAMPLE`](/curation/sampling) when the result is still larger -than you want, and [aggregate](/curation/aggregate) to check the balance of what -you ended up with. +The result contains documents **10, 20 and 30**, with labels **cat, dog and +cat**. Document 40 is excluded because this particular task requires a label. +The `row_id` tie-breaker makes the choice for document 30 deterministic; +choose a tie-breaker that fits your own data. -## Then what {#next} +The example assumes one label per document. Duplicate keys in the label table +would multiply the result rows. The [joining guide](/curation/join#labels) +shows how to check that assumption. -A curated result you want to keep is a -[materialized view](/management/materialized-view) -- stored, refreshed, and -readable by a [data loader](/training/data-loading) like any other table. +## Check the selected dataset {#check-result} + +Compare the result with the task's requirements: are the expected documents +present, are required fields populated, and is the class distribution suitable? +This example has two cats and one dog. That distribution may be appropriate +for exploration, but it does not establish a representative evaluation set. + +[Profiling](/curation/aggregate) provides the counts to inspect. +[Sampling](/curation/sampling) and [ranking](/curation/windowing) help you +adjust the selection without changing the original records. + +## Save and reuse the result {#next} + +Choose the output according to how you will use it: + +- **A fixed dataset:** save the query result in a new table with + [`CREATE TABLE AS SELECT`](/sql/ddl/table#create). Keep that output unchanged + when comparing experiments. +- **A reusable selection:** save the query as a [view](/management/view). + A view stores the query definition, so it is not a frozen evaluation set. +- **Stored results that you refresh:** use a + [materialized view](/management/materialized-view) when its supported query + and refresh behavior fit the workflow. + +Record the source versions and selection rules alongside the output. Continue +with [data loading](/training/data-loading) to use the curated data in a +training pipeline. diff --git a/docs/web/curation/join.mdx b/docs/web/curation/join.mdx index b4c198ef9..03a45326e 100644 --- a/docs/web/curation/join.mdx +++ b/docs/web/curation/join.mdx @@ -1,91 +1,133 @@ --- title: "Joining datasets" sidebarTitle: "Join" -description: "Bringing labels, metadata and scores together with the rows they describe." +description: "Combine records with labels or metadata, and check for missing or duplicate matches." icon: link keywords: ["join", "labels", "metadata", "enrichment", "namespace"] +availability: + oss: unavailable + enterprise: available + summary: >- + Enterprise supports SQL joins. --- -Enterprise Joins are SQL, and -[SQL needs a deployment](/basics/sql#sql-is-remote). +A SQL join combines records from two tables using a shared key. Use joins to +attach labels, metadata or model scores to your documents. Choose an inner join +to keep matching records, or a left join to retain records without a match. -Curation usually means combining things that were produced separately — images -in one table, human labels in another, model scores in a third. A join is how -they become one dataset. +## Example data {#example-data} -## Attaching labels {#labels} +Connect with an [Enterprise SQL client](/basics/sql). The examples use a small +document table and a separate label table. Expand the setup below to create them. + + + +`curation_documents` has six records describing four documents. +`curation_labels` has one label for each of three documents. `row_id` identifies +a record; `doc_id` identifies the document it describes. + +Run each statement separately in a scratch database or namespace. Create these +tables once for the whole section, using fresh names if they already exist. ```sql -SELECT i.id, i.embedding, l.label -FROM images AS i -JOIN labels AS l ON l.image_id = i.id +CREATE TABLE curation_documents ( + row_id BIGINT, doc_id BIGINT, version BIGINT, + source VARCHAR, caption VARCHAR, score DOUBLE +); +INSERT INTO curation_documents VALUES + (1, 10, 1, 'web', NULL, 0.80), + (2, 10, 2, 'web', 'A cat on a chair', 0.95), + (3, 20, 1, 'books', 'A dog in a field', 0.90), + (4, 30, 1, 'web', NULL, 0.70), + (5, 30, 1, 'web', 'A cat by a window', 0.85), + (6, 40, 1, 'books', 'A bird in a tree', 0.60); + +CREATE TABLE curation_labels (doc_id BIGINT, label VARCHAR); +INSERT INTO curation_labels VALUES + (10, 'cat'), (20, 'dog'), (30, 'cat'); ``` -An inner join keeps only rows that matched, which for curation is often exactly -what you want: unlabelled rows are not training data. +`score` is a sample quality score, not a search result score. Document 10 has +two versions. Document 30 has two records at the same version. Document 40 +has no label. These cases let you check the effect of each operation. -When you need to *see* what did not match, use a left join and filter: + + +## Basic join {#join-type} + +An inner join keeps matching records: ```sql -SELECT i.id -FROM images AS i -LEFT JOIN labels AS l ON l.image_id = i.id -WHERE l.image_id IS NULL +SELECT d.row_id, d.doc_id, l.label +FROM curation_documents AS d +JOIN curation_labels AS l ON l.doc_id = d.doc_id +ORDER BY d.row_id; ``` -That is your unlabelled backlog, which is the thing worth counting before you -plan an annotation round. +This returns **5 rows**: both records for document 10, one for document 20, +and both records for document 30. To produce one row per document, +[deduplicate the source records](/curation/windowing#dedup). -## Across namespaces {#namespaces} +### Find records without a match -Tables in different [namespaces](/management/namespace) join by their dotted -paths — the catalog is one catalog, so there is nothing to federate: +A left join keeps every record from the left table and fills missing right-side +values with `NULL`. Add `WHERE l.doc_id IS NULL` to return only unlabeled documents: ```sql -SELECT i.id, u.team -FROM prod.media.images AS i -JOIN prod.reference.uploaders AS u ON u.id = i.uploader_id +SELECT DISTINCT d.doc_id +FROM curation_documents AS d +LEFT JOIN curation_labels AS l ON l.doc_id = d.doc_id +WHERE l.doc_id IS NULL +ORDER BY d.doc_id; ``` -## Joining a search result {#search} +The result is **document 40**. You might send it for annotation, retain it for +an unlabeled-data workflow, or exclude it from a task that requires labels. -Full-text search is a table function, so its result is a relation and joins like -any other. This is how you ask which categories a query matches, rather than -just which rows: +## Handle duplicate keys {#labels} + +This example joins `curation_documents.doc_id` to `curation_labels.doc_id`. +First check whether the label table contains more than one row per document: ```sql -SELECT c.name, count(*) AS hits -FROM fts('documents', 'puppy') AS m -JOIN categories AS c ON c.id = m.category_id -GROUP BY c.name -ORDER BY hits DESC +SELECT doc_id, count(*) AS labels +FROM curation_labels +GROUP BY doc_id +HAVING count(*) > 1; ``` - -Vector search has no SQL form, so a similarity result cannot be joined this way. -Run the search in an SDK, then join over its output — or write the result to a -table first. - +The example returns no rows. If your data returns duplicates, decide whether +they represent valid multiple labels or records that need consolidation. +A join does not enforce one-to-one matching. -## Making it fast {#performance} +## Join tables across namespaces {#namespaces} -Two things dominate join cost on a lakehouse table. +Use a table's qualified path to address it in another +[namespace](/management/namespace). For example, a table named `images` in +`prod.media` is addressed as `prod.media.images`. The same matching-key and +join-type rules apply; your identity needs access to the source objects. -**Index the join key.** A [scalar index](/indexing/scalar-index) on the key lets -the planner look rows up instead of scanning for them. Check with -[`EXPLAIN`](/sql/dql/explain): an indexed key shows a `ScalarIndexQuery`, an -unindexed one shows a `LanceScan` feeding the join. +## Combine search results with metadata {#search} -**Do not carry wide columns through.** Select the columns you need before -joining, not after. Embeddings and blobs are the expensive part of a multimodal -table, and a join can materialize them many more times than the result has rows. +A [SQL full-text search](/sql/dql/fts) returns rows that can be joined to other +tables. Join using an identifier retained in the search result, then check +whether the metadata has one or several matches per result. -```sql -SELECT i.id, l.label -FROM (SELECT id, uploader_id FROM images) AS i -JOIN labels AS l ON l.image_id = i.id -``` +For an SDK search, retain the matching IDs and combine them with metadata in +your application or save the selected records to a table for further SQL work. +See [SQL search interfaces](/basics/sql#no-vector-sql) for supported query forms. -## Reference {#reference} +## Join performance {#performance} -The join types and their semantics are in [Joins](/sql/dql/join). +Check the row count, unmatched keys and duplicate matches before using a +joined dataset. Select only the columns needed for the next step, especially +when source records include large media or embedding columns. + +Use [Explain a query plan](/search/explain) to see how the query reads and +combines data. Check the plan to see whether the query uses indexes and how it performs the +join. An index on a key does not guarantee that the join will use it. + +## Save the result or continue curating {#reference} + +Keep the query for reuse or [save the selected data](/curation#next). +See [Joins](/sql/dql/join) for join types and exact SQL behavior. diff --git a/docs/web/curation/sampling.mdx b/docs/web/curation/sampling.mdx index 4e8b000cd..643b06bdf 100644 --- a/docs/web/curation/sampling.mdx +++ b/docs/web/curation/sampling.mdx @@ -1,98 +1,172 @@ --- -title: "Sampling a dataset" -sidebarTitle: "Sampling" -description: "Drawing a subset you can afford to work with, and fixing it when it has to be reproducible." +title: "SQL sampling" +sidebarTitle: "SQL Sampling" +description: "Choose a sample for exploration or evaluation and preserve the selected records when reproducibility matters." icon: dice keywords: ["sampling", "tablesample", "subset", "eval set", "reproducible", "shuffle"] +availability: + oss: unavailable + enterprise: available + summary: >- + Enterprise supports SQL `TABLESAMPLE`. For sampling in OSS and Enterprise through the Python SDK, see [Sample without SQL](#training). --- -Most curation starts with a subset. A question you can answer in ten seconds on -1% of the data is worth asking twenty times; the same question on the whole -table is worth asking once. +import { PySampleDataset } from '/snippets/curation_sampling.mdx'; -## A quick look {#quick} +Sampling selects records from a larger table. Use it to explore unfamiliar +data, run a smaller experiment, or prepare an evaluation dataset. Enterprise +SQL provides `TABLESAMPLE` for selecting a percentage or number of rows. -Enterprise +This page covers Enterprise SQL sampling and a complete +[Python SDK example](#training) that runs locally. + +## Example data {#example-data} + +Connect with an [Enterprise SQL client](/basics/sql). The examples use a small +document table and a separate label table. Expand the setup below to create them. + + + +`curation_documents` has six records describing four documents. +`curation_labels` has one label for each of three documents. `row_id` identifies +a record; `doc_id` identifies the document it describes. + +Run each statement separately in a scratch database or namespace. Create these +tables once for the whole section, using fresh names if they already exist. ```sql -SELECT * FROM images TABLESAMPLE (1) -- 1 percent -SELECT * FROM images TABLESAMPLE (500 ROWS) -- 500 rows +CREATE TABLE curation_documents ( + row_id BIGINT, doc_id BIGINT, version BIGINT, + source VARCHAR, caption VARCHAR, score DOUBLE +); +INSERT INTO curation_documents VALUES + (1, 10, 1, 'web', NULL, 0.80), + (2, 10, 2, 'web', 'A cat on a chair', 0.95), + (3, 20, 1, 'books', 'A dog in a field', 0.90), + (4, 30, 1, 'web', NULL, 0.70), + (5, 30, 1, 'web', 'A cat by a window', 0.85), + (6, 40, 1, 'books', 'A bird in a tree', 0.60); + +CREATE TABLE curation_labels (doc_id BIGINT, label VARCHAR); +INSERT INTO curation_labels VALUES + (10, 'cat'), (20, 'dog'), (30, 'cat'); ``` -A bare number is a percentage; add `ROWS` for a count. Use it while you are -exploring — the cost of the query drops with the sample, and so does the cost -of being wrong about what you wanted to ask. +`score` is a sample quality score, not a search result score. Document 10 has +two versions. Document 30 has two records at the same version. Document 40 +has no label. These cases let you check the effect of each operation. -## Rows or fragments {#methods} + + +## Basic sampling {#quick} + +A percentage selects a fraction of the source; `ROWS` requests a row count: ```sql -SELECT * FROM images TABLESAMPLE BERNOULLI (10) -- rows -SELECT * FROM images TABLESAMPLE SYSTEM (10) -- fragments +SELECT row_id, doc_id FROM curation_documents TABLESAMPLE (50); +SELECT row_id, doc_id FROM curation_documents TABLESAMPLE (3 ROWS); ``` -`BERNOULLI` considers each row independently and reads across the whole table. -`SYSTEM` picks whole Lance fragments and takes the rows in them — far less I/O, -because it skips files rather than rows. +A percentage sample does not guarantee an exact number of rows. Use a count +when the desired sample size is the main constraint, and inspect the result +rather than assuming it contains every category you need. -The trade is uniformity. A fragment sample is only as representative as your -write order. If rows arrived grouped by source, by date, or by label — which is -usual — then a fragment sample inherits that grouping. +`TABLESAMPLE` requires a table that is not WAL-backed. See the +[sampling reference](/sql/dql/sampling#refused) for this restriction; do not +change a production table's write configuration just to try an example. - -Reach for `SYSTEM` when you want a cheap look, and `BERNOULLI` when the number -you compute has to be defensible. Omitting the method gives `BERNOULLI`. - +## Sampling methods {#methods} -## A fixed evaluation set {#fixed} +Choose a method according to how records should be selected: -Without a seed, every execution resamples. That is fine for exploring and wrong -for evaluation — a metric computed on different rows each run is not comparable -with itself. +| Method | Selection | When to use it | +| --- | --- | --- | +| `BERNOULLI` (default) | Select individual rows | When records should be sampled independently of their storage fragments | +| `SYSTEM` | Select whole Lance fragments | When reducing reads matters and grouping by fragment is acceptable | ```sql -SELECT * FROM images TABLESAMPLE (10) REPEATABLE (42) +SELECT row_id, doc_id +FROM curation_documents TABLESAMPLE BERNOULLI (50); + +SELECT row_id, doc_id +FROM curation_documents TABLESAMPLE SYSTEM (50); ``` -The same seed against the same data gives the same rows, which is what makes a -sample usable as a held-out set. If the underlying data changes, so does the -draw — when you need the set frozen against that too, materialize it: +If write order groups records by label, date or source, a fragment sample can +reflect that grouping. On a tiny table stored in one fragment, it may include +all records or none. Row sampling also needs an adequate sample size to +represent rare groups; check that the resulting sample covers the groups you need. + +## Repeatable samples {#fixed} + +A seed requests a repeatable draw: ```sql -CREATE TABLE eval_set AS -SELECT * FROM images TABLESAMPLE (10) REPEATABLE (42) +SELECT row_id, doc_id +FROM curation_documents TABLESAMPLE (50) REPEATABLE (42); ``` -A [materialized view](/management/materialized-view) is the other option when -you want it to track the source on a refresh rather than stay frozen. - -## Sampling both sides of a join {#joins} +Keep the source data and sampling configuration fixed when comparing draws. +For a lasting evaluation set, save the selected records rather than relying +on a seed to preserve membership as the source changes: ```sql -SELECT * FROM a TABLESAMPLE (10) JOIN b TABLESAMPLE (10) ON a.id = b.id +CREATE TABLE curation_eval AS +SELECT * FROM curation_documents TABLESAMPLE (50) REPEATABLE (42); ``` -Each side samples independently, so a row survives only if both of its sides -were drawn. Sampling 10% of each side does **not** give 10% of the join — it -gives roughly 1%. Sample once, after the join, unless you specifically want the -smaller result. +Create this output once with a fresh name and keep it unchanged. Record the +source version and selection policy. Sampling records independently can place +revisions of the same document into different splits; decide the +[grouping policy](/curation/windowing#splits) before using the result for +training or evaluation. -## Shuffling for training {#training} +## Sampling with joins {#joins} -For a training loop rather than a query, the SDKs read a table in a shuffled -order without SQL at all: +Sampling both input tables independently changes which pairs can match. +For a one-to-one match, independent 10% row samples retain a given pair with +about 1% probability, not 10%. -```python -from lancedb.permutation import Permutation +To sample documents while retaining their available labels, sample only the +document table: -permutation = Permutation.identity(table) -dataloader = torch.utils.data.DataLoader(permutation) +```sql +SELECT curation_documents.row_id, curation_documents.doc_id, l.label +FROM curation_documents TABLESAMPLE (50) +JOIN curation_labels AS l ON l.doc_id = curation_documents.doc_id; ``` -This is the right tool for epoch-level shuffling, where you want every row -eventually and only the order varies. `TABLESAMPLE` is for when you want fewer -rows. See [Training](/training/data-loading). +This samples source records, not distinct documents. If you need to sample the +result of a join or deduplication, first save that result as a table suitable +for sampling. `TABLESAMPLE` cannot be applied directly to a subquery result. -## Reference {#reference} +## Sample without SQL {#training} -The full clause, its options and what it refuses are in -[Sampling](/sql/dql/sampling). +The Python SDK provides a permutation builder for selecting and ordering rows +in OSS and Enterprise. A split selects a subset; a shuffle changes the order +in which records are read. `Permutation.identity(table)` retains stored order. + + + +Install the Python SDK with `pip install lancedb`, save the program as +`sample_dataset.py`, and run `python sample_dataset.py`. It creates a temporary +local database with ten rows and reads a random split containing three rows. +The temporary database is removed when the program finishes. + + +{PySampleDataset} + + +The program prints three distinct IDs. It uses a fixed seed, but does not +promise the same IDs across SDK versions or changes to the source dataset. + + + +For shuffled training reads, worker configuration and data loaders, see +[Loading data for model training](/training/data-loading). + +## Related guides {#reference} + +Count the sampled rows and [inspect their distribution](/curation/aggregate) +before using them. Preserve the selected data when the exact membership matters. +See [SQL sampling](/sql/dql/sampling) for syntax and restrictions. diff --git a/docs/web/curation/windowing.mdx b/docs/web/curation/windowing.mdx index fbb3cbbf2..fa84ed5fe 100644 --- a/docs/web/curation/windowing.mdx +++ b/docs/web/curation/windowing.mdx @@ -1,124 +1,162 @@ --- title: "Ranking and deduplication" sidebarTitle: "Windowing" -description: "Picking one row per thing, ranking within a group, and splitting a dataset." +description: "Choose records with explicit ranking rules and deterministic tie handling." icon: chart-simple keywords: ["window", "deduplication", "row_number", "rank", "train test split", "per class"] +availability: + oss: unavailable + enterprise: available + summary: >- + Enterprise supports SQL window functions. --- -Enterprise Window functions are SQL, and -[SQL needs a deployment](/basics/sql#sql-is-remote). +Window functions calculate values across related rows while keeping each row +in the result. Use them to rank documents, keep one record per item, or compare +a record with the one before it. -A window function computes across a set of related rows and returns a value -**per row** — nothing is collapsed. That is what makes it the tool for "which of -these do I keep", where an aggregate would throw away the rows you are choosing -between. +## Example data {#example-data} -## One row per thing {#dedup} +Connect with an [Enterprise SQL client](/basics/sql). The examples use a small +document table and a separate label table. Expand the setup below to create them. -The most common curation query there is. You have several rows per document — -revisions, re-uploads, repeated annotations — and want one. + + +`curation_documents` has six records describing four documents. +`curation_labels` has one label for each of three documents. `row_id` identifies +a record; `doc_id` identifies the document it describes. + +Run each statement separately in a scratch database or namespace. Create these +tables once for the whole section, using fresh names if they already exist. + +```sql +CREATE TABLE curation_documents ( + row_id BIGINT, doc_id BIGINT, version BIGINT, + source VARCHAR, caption VARCHAR, score DOUBLE +); +INSERT INTO curation_documents VALUES + (1, 10, 1, 'web', NULL, 0.80), + (2, 10, 2, 'web', 'A cat on a chair', 0.95), + (3, 20, 1, 'books', 'A dog in a field', 0.90), + (4, 30, 1, 'web', NULL, 0.70), + (5, 30, 1, 'web', 'A cat by a window', 0.85), + (6, 40, 1, 'books', 'A bird in a tree', 0.60); + +CREATE TABLE curation_labels (doc_id BIGINT, label VARCHAR); +INSERT INTO curation_labels VALUES + (10, 'cat'), (20, 'dog'), (30, 'cat'); +``` + +`score` is a sample quality score, not a search result score. Document 10 has +two versions. Document 30 has two records at the same version. Document 40 +has no label. These cases let you check the effect of each operation. + + + +## Keep one record per document {#dedup} + +`row_number()` numbers rows within each group. `PARTITION BY` defines the group, +and `ORDER BY` determines which record comes first. This query selects the +latest version of each document, using `row_id` to break ties: ```sql WITH ranked AS ( - SELECT *, - row_number() OVER (PARTITION BY doc_id ORDER BY version DESC) AS rn - FROM revisions + SELECT *, row_number() OVER ( + PARTITION BY doc_id ORDER BY version DESC, row_id DESC + ) AS rn + FROM curation_documents ) -SELECT * FROM ranked WHERE rn = 1 +SELECT row_id, doc_id, version, caption +FROM ranked +WHERE rn = 1 +ORDER BY doc_id; ``` -`PARTITION BY` says what "per thing" means; `ORDER BY` says which one wins. The -outer query is not optional — a window function cannot appear in `WHERE`, -because windows are computed after filtering, so the predicate would have -nothing to read. +This returns row IDs **2, 3, 5 and 6**. For document 30, rows 4 and 5 have the +same version; the example deliberately chooses the larger `row_id`. +Without a complete ordering, the selected row among ties is not determined by +the query. - -Swap `row_number()` for `rank()` when ties should both survive. `row_number()` -always picks exactly one; `rank()` gives tied rows the same rank, so `rank = 1` -can return several. - +Use `rank()` instead of `row_number()` when tied records should all survive, +and order only by the fields that define a tie. Adding a unique tie-breaker +would make every rank distinct. The outer query filters the calculated rank +because window functions are evaluated after the inner query's `WHERE` clause. -## A fixed number per class {#per-group} +## Select the highest-scoring records per class {#per-group} -Class imbalance found by [profiling](/curation/aggregate#balance) is fixed here: -rank within each class, then take the top N. +Deduplicate first, then rank the labeled documents within each class: ```sql -WITH ranked AS ( - SELECT *, row_number() OVER (PARTITION BY label ORDER BY score DESC) AS rn - FROM images +WITH latest AS ( + SELECT *, row_number() OVER ( + PARTITION BY doc_id ORDER BY version DESC, row_id DESC + ) AS version_rank + FROM curation_documents +), ranked AS ( + SELECT d.doc_id, l.label, d.score, + row_number() OVER ( + PARTITION BY l.label ORDER BY d.score DESC, d.doc_id + ) AS class_rank + FROM latest AS d + JOIN curation_labels AS l ON l.doc_id = d.doc_id + WHERE d.version_rank = 1 ) -SELECT * FROM ranked WHERE rn <= 1000 +SELECT doc_id, label, score +FROM ranked +WHERE class_rank <= 1 +ORDER BY label; ``` -That gives at most 1,000 rows per label, taking the highest-scoring ones. Order -by something arbitrary but stable if you want an unbiased draw instead, or -combine with [sampling](/curation/sampling) to randomize within each class. +The result is **document 10 for cat and document 20 for dog**. Change the cap +from `1` to the maximum number you want per class. A class with fewer records +cannot supply the full cap. -## Splitting a dataset {#splits} +This selects by score, not at random. For a random subset, use +[sampling](/curation/sampling) instead of ordering by score or ID. -`ntile` divides each partition into equal buckets, which is a train/test split -that stays balanced per class: +## Training and evaluation splits {#splits} + +Ranking can divide ordered records into groups, but ordering by ID does not +make those groups random or representative. `ntile()` produces approximately +equal-sized buckets within a partition; small classes may not populate every +bucket. + +Choose the split policy for the task. Keep related records, such as revisions +of one document or frames from one video, in the same split to avoid leakage. +A time-based evaluation may instead require a cutoff date. Save the final +assignments or resulting datasets so later updates do not silently change the +evaluation population. + +## Compare neighboring records {#offset} + +`lag()` and `lead()` read neighboring rows in the specified order. For example, +inspect how scores change across a document's records: ```sql -WITH split AS ( - SELECT *, ntile(10) OVER (PARTITION BY label ORDER BY id) AS bucket - FROM images -) -SELECT * FROM split WHERE bucket <= 8 -- train +SELECT doc_id, row_id, score, + lag(score) OVER ( + PARTITION BY doc_id ORDER BY version, row_id + ) AS previous_score +FROM curation_documents +ORDER BY doc_id, version, row_id; ``` -Buckets 9 and 10 are your validation and test sets. Because the partition is by -label, each split keeps the class proportions of the whole. +The first record in each document has no previous score. In this dataset, +row 2 follows a score of 0.80 and row 5 follows a score of 0.70. - -Order by something stable. Ordering by a value that changes — a score you -recompute, a timestamp that gets updated — means the split changes with it, and -a row can move from train to test between runs. - + +## Window query performance {#cost} -## Comparing a row to its neighbours {#offset} +Window operations can require sorting. Filter out records you do not need +before ranking, and inspect the +[query plan](/search/explain) when cost matters. -`lag` and `lead` read the previous and next row in the window, which is how you -find gaps and changes without a self-join: +For cumulative counts, sums and other running summaries, use an aggregate +with an explicit window order and frame. The +[window reference](/sql/dql/window#window) covers those expressions. -```sql -SELECT - id, - created_at, - created_at - lag(created_at) OVER (ORDER BY created_at) AS gap -FROM events -``` +## Continue with the curated data {#reference} -Useful for spotting ingestion gaps, duplicate timestamps, and runs of identical -values that suggest a broken upstream job. - -## Running totals {#running} - -Any [aggregate](/curation/aggregate) becomes a window function with `OVER`, -which is how cumulative coverage is expressed: - -```sql -SELECT - label, - count(*) AS n, - sum(count(*)) OVER (ORDER BY count(*) DESC) AS cumulative -FROM images -GROUP BY label -``` - -Reading down the `cumulative` column tells you how many classes cover most of -your data — usually far fewer than you expect. - -## Cost {#cost} - -Window functions sort within each partition, so on a large table they are among -the more expensive things you can ask for. Filter first, partition on something -selective, and check the plan with -[Explain Analyze](/search/explain). - -## Reference {#reference} - -The full window syntax, framing, and function list are in -[Window functions](/sql/dql/window). +[Profile the selected records](/curation/aggregate) and +[save the result](/curation#next) for downstream use. +See [Window functions](/sql/dql/window) for the full syntax and function list. diff --git a/docs/web/deployment/monitoring.mdx b/docs/web/deployment/monitoring.mdx index 6a66eb0ed..5ad56ce1c 100644 --- a/docs/web/deployment/monitoring.mdx +++ b/docs/web/deployment/monitoring.mdx @@ -11,10 +11,10 @@ LanceDB emits internal metrics (currently object store request counts, bytes tra The bridge is available in the Python and TypeScript SDKs. It is a thin wrapper over LanceDB's `metrics` recorder; your application supplies and configures the OpenTelemetry SDK. -This page covers LanceDB OSS instrumentation. LanceDB Enterprise clusters emit their own metrics from the server side — see [Enterprise monitoring](#enterprise) below. +This page covers LanceDB OSS instrumentation. LanceDB Enterprise clusters emit their own metrics from the server side. See [Enterprise monitoring](#enterprise) below. -## What you get {#what-you-get} +## Available metrics {#what-you-get} Once instrumented, LanceDB registers one observable instrument per metric on your `MeterProvider`. The current catalog covers the object store layer: @@ -22,8 +22,10 @@ Once instrumented, LanceDB registers one observable instrument per metric on you |--------|------|-------------| | `lance_object_store_requests_total` | Counter | Total object store requests, labelled by `operation` and `base` (store scheme). | | `lance_object_store_request_duration_seconds` | Histogram | Request latency in seconds. | -| `lance_object_store_bytes_transferred_total` | Counter | Bytes read from or written to the store. | -| `lance_object_store_retryable_responses_total` | Counter | Requests that returned a retryable error (throttles, transient failures). | +| `lance_object_store_request_bytes_total` | Counter | Bytes read from or written to the store. | +| `lance_object_store_errors_total` | Counter | Object store requests that failed. | +| `lance_object_store_retryable_responses_total` | Counter | Retryable HTTP responses (5xx, 429 and 408) from a store reached over HTTP. | +| `lance_object_store_throttle_total` | Counter | Throttle responses (429 and 503) from a store reached over HTTP. | | `lance_object_store_in_flight_requests` | Gauge | Currently outstanding object store requests. | The recorder is process-global and pull-based: your configured `MetricReader` collects on its own schedule, so there is no hot-path overhead beyond the atomic aggregation that LanceDB does anyway. @@ -37,7 +39,7 @@ The recorder is process-global and pull-based: your configured `MetricReader` co Install LanceDB with the `otel` extra to pull in the OpenTelemetry API, plus an OpenTelemetry SDK of your choice. The SDK is intentionally not bundled, so you configure it and its readers and exporters however your platform expects. ```bash -pip install "lancedb[otel]" opentelemetry-sdk +pip install "lancedb[otel]" opentelemetry-sdk opentelemetry-exporter-otlp-proto-grpc ``` Call `instrument_lancedb_metrics()` once at startup, before opening any tables. It returns `True` when the recorder is installed and instruments are registered. @@ -70,7 +72,7 @@ If you omit `meter_provider`, LanceDB uses the global provider returned by `open ## TypeScript {#typescript} -The Node SDK depends on `@opentelemetry/api` directly, so no extra install step is needed to expose the entry point. You still need an OpenTelemetry SDK to actually export. +Install the OpenTelemetry SDK and exporter in your application: ```bash npm install @opentelemetry/sdk-metrics @opentelemetry/exporter-metrics-otlp-grpc @@ -93,7 +95,7 @@ const db = await connect("s3://my-bucket/lancedb"); `instrumentLanceDbMetrics()` also accepts no arguments, in which case it uses the global provider from `@opentelemetry/api`. Calling it more than once is safe: instruments are created only on the first successful call. -## What to watch {#what-to-watch} +## Build dashboards and alerts {#what-to-watch} A few starting points for dashboards and alerts: @@ -116,7 +118,7 @@ export them with data enrichment -- no client-side instrumentation needed. [Contact support](/deployment/support) for the comprehensive metric list and help integrating them with your monitoring dashboard. -## Where to go next {#where-to-go-next} +## Related guides {#where-to-go-next} diff --git a/docs/web/deployment/storage.mdx b/docs/web/deployment/storage.mdx index fd8550ea6..4994e8f0c 100644 --- a/docs/web/deployment/storage.mdx +++ b/docs/web/deployment/storage.mdx @@ -1,11 +1,24 @@ --- -title: "Storage" -sidebarTitle: "Storage" -description: "Choosing and configuring where LanceDB stores its data -- how you deploy an embedded database." +title: "Embedded storage configuration" +sidebarTitle: "Embedded Storage" +description: "Choose and configure local disk or object storage for tables accessed by the embedded LanceDB OSS library." icon: database keywords: ["storage", "s3", "gcs", "azure", "object store", "configuration", "deploy"] +availability: + oss: available + enterprise: unavailable + summary: >- + The embedded library chooses and configures its storage backend from the + connection URI and `storage_options`. In LanceDB Enterprise, storage is part of + the deployment's own configuration instead. --- +This page is about the embedded library, where your connection chooses and +configures the storage backend. A LanceDB Enterprise deployment sets up its own +storage when it is deployed, so a `db://` connection passes none of these +options except an Azure storage account name -- see +[Deployment](/enterprise/deployment) for how a deployment is set up. + LanceDB's storage layer is built on modular, disk-first components. That design makes it flexible enough to run across local NVMe, EBS, EFS, and any object store that exposes an S3-compatible API. It also supports region-specific backends such as [Tencent COS](/storage/configuration#tencent-cos) and cache-acceleration layers such as [GooseFS](/storage/configuration#goosefs). Choosing a backend is a balance between latency, scalability, cost, and operational complexity. Use this guide to pick the right fit for your workload. @@ -176,9 +189,9 @@ Table-level `storage_options` inherit every key from the connection and override -**Inspect the effective options** +**Inspect a table's own options** -On `AsyncTable`, `await table.initial_storage_options()` returns the options the table was opened with, and `await table.latest_storage_options()` returns the current options after any provider-driven refresh. The deprecated `table.storage_options()` method will be removed in a future release. +On `AsyncTable`, `await table.initial_storage_options()` returns the options passed when the table was opened, and `await table.latest_storage_options()` returns them after any provider-driven refresh. Neither includes what the table inherits from its connection's `storage_options`, which still apply. The deprecated `table.storage_options()` method will be removed in a future release. #### General object store options {#general-object-store-options} diff --git a/docs/web/deployment/support.mdx b/docs/web/deployment/support.mdx index f758cab00..5e35d9131 100644 --- a/docs/web/deployment/support.mdx +++ b/docs/web/deployment/support.mdx @@ -9,11 +9,10 @@ keywords: ["support", "help", "discord", "community"] ## LanceDB OSS {#oss} For a quick question, the -[Discord community](https://discord.gg/AUEWnJ7Txb) is fastest -- ask under the -relevant language channel for Python, TypeScript, or Rust, and you reach both -the community and the engineering team. +[Discord community](https://discord.gg/AUEWnJ7Txb) has channels for +Python, TypeScript and Rust where you can ask the community for help. -For anything worth keeping searchable for the next person, use GitHub instead: +Use GitHub for discussions and issues that others can search and follow: | | | | --- | --- | diff --git a/docs/web/docs.json b/docs/web/docs.json index 4c002b0a5..a8fb44007 100644 --- a/docs/web/docs.json +++ b/docs/web/docs.json @@ -63,10 +63,6 @@ "source": "/feature-engineering/computed-columns", "destination": "/feature-engineering/add-computed-column" }, - { - "source": "/feature-engineering/functions", - "destination": "/management/function" - }, { "source": "/tables/create", "destination": "/ingestion" @@ -163,10 +159,6 @@ "source": "/storage/monitoring", "destination": "/deployment/monitoring" }, - { - "source": "/enterprise/deployment", - "destination": "/enterprise/deployment/index" - }, { "source": "/search/sql", "destination": "/sql" @@ -253,6 +245,8 @@ "group": "Getting Started", "pages": [ "index", + "basics/offerings", + "basics/connect", "quickstart", "architecture", "basics/sdk", @@ -270,7 +264,8 @@ "basics/jobs", "tables/consistency", "basics/guarantees", - "basics/limits" + "basics/limits", + "basics/naming" ] }, { @@ -333,8 +328,7 @@ "management/table", "tables/branching", "management/materialized-view", - "management/view", - "management/function" + "management/view" ] }, { @@ -384,7 +378,6 @@ "sql/ddl/view", "sql/ddl/materialized-view", "sql/ddl/index", - "sql/ddl/function", "sql/ddl/job" ], "icon": "sitemap" diff --git a/docs/web/embedding/index.mdx b/docs/web/embedding/index.mdx index e3568604b..9f1531c05 100644 --- a/docs/web/embedding/index.mdx +++ b/docs/web/embedding/index.mdx @@ -3,6 +3,13 @@ title: "Managing Embeddings" sidebarTitle: "Managing Embeddings" description: "Use the embedding API in LanceDB -- registry, functions, schemas, and multi-language SDK support." icon: "bars" +availability: + oss: available + enterprise: available + summary: >- + Embedding functions run in your process, whether the table is local or on a + deployment, which stores their configuration but never runs them. Python + and TypeScript also embed query text; in Rust you embed queries yourself. --- import { @@ -30,8 +37,9 @@ the data, so vectors that are close to each other are considered similar. LanceDB provides an embedding function registry in OSS as well as its Enterprise versions ([see below](#embeddings-in-lancedb-enterprise)) -that automatically generates vector embeddings during data ingestion. Automatic query-time embedding -generation is available in LanceDB OSS, with SDK-specific query ergonomics. The API abstracts +that automatically generates vector embeddings during data ingestion. Query text is embedded +automatically too, in the client, for local and Enterprise tables alike, with SDK-specific query +ergonomics. The API abstracts embedding generation, allowing you to focus on your application logic. ## Embedding Registry {#embedding-registry} @@ -41,6 +49,9 @@ Once configured, the embedding function will automatically generate embeddings w into the table. Query-time behavior depends on SDK: Python/TypeScript can query with text directly, while Rust examples typically compute query embeddings explicitly before vector search. +In the Rust examples, replace `your-openai-api-key` with your OpenAI API key. In +application code, load credentials from environment variables or a secrets manager. + {PyOpenaiEmbeddings} diff --git a/docs/web/enterprise/authentication.mdx b/docs/web/enterprise/authentication.mdx index f41603fdf..4d04120fc 100644 --- a/docs/web/enterprise/authentication.mdx +++ b/docs/web/enterprise/authentication.mdx @@ -4,14 +4,15 @@ sidebarTitle: "Authentication" description: "Authentication modes for LanceDB Enterprise: API keys and OAuth credentials." icon: key keywords: ["authentication", "oauth", "api key", "azure managed identity", "client credentials"] +availability: + oss: unavailable + enterprise: available + summary: >- + Requests to a deployment carry an API key or an OAuth token, whichever it is + configured to accept. Embedded LanceDB has no server to sign in to; see + [Architecture](/architecture). --- -Enterprise - -This page describes a LanceDB Enterprise capability. The embedded form -of LanceDB has no server to provide it -- see -[Architecture](/architecture) for what differs between the two. - A deployment authenticates every request before [authorization](/governance/authorization) decides what it may do, against a `db://` remote table, in one of two ways: @@ -50,6 +51,10 @@ from an OIDC issuer and attach them to every request. Token acquisition, caching, and refresh are handled inside the client -- your application code only provides the configuration. +A deployment accepts these tokens only when its administrator has configured +it with your identity provider. Ask them for the issuer URL, client ID and +scopes to use. + In Python, OAuth is supported through `lancedb.connect_async`. The synchronous `connect` entry point continues to use API key authentication for `db://` diff --git a/docs/web/enterprise/index.mdx b/docs/web/enterprise/index.mdx index ed4d232c1..a601e850d 100644 --- a/docs/web/enterprise/index.mdx +++ b/docs/web/enterprise/index.mdx @@ -21,9 +21,13 @@ What it adds over the embedded form: | | | | --- | --- | | [SQL](/sql) | an Arrow Flight SQL endpoint | -| [Functions](/management/function) | Python running on workers | | [Jobs](/basics/jobs) | server-side work with ids, status and events | | [Authorization](/governance/authorization) | per-object grants | | Scale | a cluster rather than your process | +{/* +Draft for the Functions release. Add this row to the table above, after SQL: +| [Functions](/management/function) | Python running on workers | +*/} + See [Architecture](/architecture) for the two shapes side by side. diff --git a/docs/web/feature-engineering/add-computed-column.mdx b/docs/web/feature-engineering/add-computed-column.mdx index 571797d4e..331918715 100644 --- a/docs/web/feature-engineering/add-computed-column.mdx +++ b/docs/web/feature-engineering/add-computed-column.mdx @@ -4,13 +4,18 @@ sidebarTitle: "Computed Columns" description: "Declare a column from a SQL expression. Nothing is evaluated until you refresh it." icon: table-columns keywords: ["computed columns", "add_columns", "transforms", "schema"] +availability: + oss: available + enterprise: available + summary: >- + Both offerings support computed columns. Enterprise requires the deployment to enable them; the embedded library evaluates them in your process. --- A computed column is declared with an expression and starts with no values. Declaring it is cheap on any table size, because nothing is evaluated until [a refresh](/feature-engineering/refresh-column) asks for it. -## Declaring one {#declare} +## Add a computed column {#declare} ```python import lancedb @@ -26,14 +31,17 @@ That commits a new table version with the column present and empty. It is an ordinary table version, so the usual [versioning](/management/table#versioning) applies -- including that you can travel back to before the column existed. -## `computed` and `transforms` are different {#computed-vs-transforms} +## Compute now or on refresh {#computed-vs-transforms} `add_columns` takes either, and the difference matters: - **SQL expressions in `transforms`** are evaluated *now*, for every row, as part of the call. Good for a small table, or a column you want populated - immediately. A `FunctionApplication` passed through `transforms` instead - declares empty Function-bound columns; a refresh fills them. + immediately. + {/* Draft for the Functions release: + A `FunctionApplication` passed through `transforms` instead + declares empty Function-bound columns, which a refresh fills; that needs a + deployment with Functions enabled, and the embedded library refuses it. */} - **`computed`** stores the expression and evaluates nothing. The column is committed empty and filled later by [`refresh_column`](/feature-engineering/refresh-column). @@ -48,8 +56,8 @@ the table, including another computed column: ```python table.add_columns(computed={ - "power_score": "(stats.magic + stats.strength) / 2.0", - "is_strong": "power_score > 3.0", + "power_score": "cast(x as double) / 2.0", + "is_strong": "power_score > 0.5", }) ``` @@ -59,12 +67,22 @@ declared earlier in the same mapping, but not one declared after it. The functions available are the ones the [SQL dialect](/sql/functions) documents. -## Its inputs become load-bearing {#input-constraints} +A literal in the expression has to fit the column it meets: `x / 2.0` is +refused when `x` is an integer column, so write `cast(x as double) / 2.0`. + + +In LanceDB Enterprise, keep the expression to top-level columns. A column +refresh can fail on an expression that reads a field inside a struct, such as +`stats.magic + stats.strength`, which the embedded library computes. Try a new +expression on a small table first. + + +## Change columns used by an expression {#input-constraints} While a computed column's declaration references another column, that column -cannot be renamed, retyped, or dropped. The declaration is part of the schema, -and LanceDB will not let you invalidate it out from under itself. +cannot be renamed, retyped, or dropped. Drop the computed column first +if you need to change one of its inputs. This is worth knowing before deriving a column from something you expect to @@ -74,18 +92,33 @@ reshape later. Dropping the computed column releases the constraint. Enterprise +When the value cannot be expressed in SQL -- an embedding, a model call, an +image decode -- [Geneva](/geneva), the feature engineering package for LanceDB +Enterprise, fills the column with a Python function instead. + +{/* +Draft for the Functions release. Replaces the paragraph above: + When the value cannot be expressed in SQL -- an embedding, a model call, an image decode -- the column is filled by a registered [function](/management/function) instead of an expression. Declaring it looks -the same; what differs is what fills it. +the same; what differs is what fills it. It needs a deployment with Functions +enabled. +*/} -## What to read next {#next} +## Related guides {#next} Fill the rows that lack values. + + Fill a column with Python on LanceDB Enterprise. + + {/* + Draft for the Functions release. Replaces the Geneva card above: Fill a column with Python. + */} diff --git a/docs/web/feature-engineering/data-evolution.mdx b/docs/web/feature-engineering/data-evolution.mdx index 5f50bdcbb..f70612cf9 100644 --- a/docs/web/feature-engineering/data-evolution.mdx +++ b/docs/web/feature-engineering/data-evolution.mdx @@ -4,6 +4,12 @@ sidebarTitle: "Data Evolution" description: "Derive new columns from data already in the table, without rewriting what is already there." icon: wand-magic-sparkles keywords: ["data evolution", "derived columns", "add_columns", "zero copy", "feature engineering"] +availability: + oss: available + enterprise: available + summary: >- + Adding columns computed from SQL expressions works in the embedded library + and in LanceDB Enterprise. --- import { @@ -18,20 +24,23 @@ import { RsAddFeatureColumnsSql as RsAddFeatureColumnsSql, } from '/snippets/tables.mdx'; -Data evolution is adding **values** derived from what a table already holds. -[Schema evolution](/management/table#schema) changes the table's shape; this fills the -shape with something new. +Add derived columns to calculate features from data already in a table. +For example, compute a product score from existing fields without reading +and rewriting the table in your application. -The reason it is cheap is the format. Adding a derived column writes only that -column -- the existing data, which is the expensive part on a multimodal table, -is never rewritten. See [why that works](/basics/lance#evolution). +Lance can write the new columns without rewriting the existing columns. +This is useful when the table also contains large images, video or embeddings. +See [Lance format](/basics/lance) for the storage model. -In SQL this is [`ALTER TABLE ... ADD COLUMN ... AS`](/sql/ddl/table#computed-column). +In SQL, on an Enterprise cluster that enables computed columns, this is [`ALTER TABLE ... ADD COLUMN ... AS`](/sql/ddl/table#computed-column). ## Set up the example table {#set-up-the-example-table} +The examples use `db`, a connection opened as in +[Connect to LanceDB](/basics/connect), and create a small product table in it: + {PySchemaAddSetup} @@ -96,16 +105,24 @@ If your transformation cannot be expressed in SQL, compute the values outside ## When SQL is not enough {#beyond-sql} -The expressions above are evaluated once, as part of the call. Two other shapes -exist when that does not fit: +The expressions above are evaluated once, as part of the call. For other workflows: - A [computed column](/feature-engineering/add-computed-column) declares the expression and leaves the column empty, so you choose when the work happens. - That is the right default on a large table. + Use it when you want to schedule the computation separately. +- On LanceDB Enterprise, [Geneva](/geneva) fills a column with Python, for + values SQL cannot express -- an embedding, a model call, an image decode. + +{/* +Draft for the Functions release. Replaces the Geneva item in the list above: - A [function](/management/function) fills a column with Python, for values SQL cannot express -- an embedding, a model call, an image decode. +*/} -## What to read next {#next} +A deployment has to enable computed columns before you use them. +{/* Draft for the Functions release: Functions need enabling there too. */} + +## Related guides {#next} diff --git a/docs/web/feature-engineering/index.mdx b/docs/web/feature-engineering/index.mdx index 4b8200d40..7fe699648 100644 --- a/docs/web/feature-engineering/index.mdx +++ b/docs/web/feature-engineering/index.mdx @@ -3,52 +3,90 @@ title: "Feature engineering" sidebarTitle: "Overview" description: "Derive columns and tables from data already in the lakehouse, without moving it somewhere else to compute." icon: layer-group -keywords: ["feature engineering", "computed columns", "materialized views", "functions", "lakehouse", "jobs"] +keywords: ["feature engineering", "computed columns", "materialized views", "lakehouse", "jobs"] --- -Feature engineering happens where the data already is. Rather than exporting a -table, computing somewhere else and writing the result back, you declare what a -column or a table should contain and let LanceDB fill it. +Use feature engineering to turn raw data into features for search, analysis +and model training. LanceDB stores derived columns and materialized views +alongside the source data, so you can reuse the results across those workflows. + +[Materialized views](/feature-engineering/materialized-views) are ordinary +[catalog objects](/basics/catalog). Create them through the SDK or SQL, +then refresh the derived data when its inputs change. + +{/* +Draft for the Functions release. Replaces the paragraph above: [Functions](/management/function) and [materialized views](/feature-engineering/materialized-views) are ordinary -[catalog objects](/basics/catalog), alongside tables and indexes -- created, -versioned, and refreshed like anything else, not a separate system bolted -alongside. +[catalog objects](/basics/catalog). Create them through the SDK or SQL, +then refresh the derived data when its inputs change. +*/} -Every operation here has a SQL form -- see [DDL](/sql/ddl/table#alter) to declare a column and [REFRESH](/sql/dml/refresh) to fill it. +In LanceDB Enterprise, every operation here also has a SQL form -- see [DDL](/sql/ddl/table#alter) to declare a column and [REFRESH](/sql/dml/refresh) to fill it. -## The shape everything shares {#the-shape} +## Computed columns and materialized views {#the-shape} -Every mechanism here separates **declaring** from **filling**, and that -separation is the point: +Computed columns and materialized views both keep derived data in the +lakehouse, but they fill it differently, and the difference decides what a +refresh costs: -- Declaring is cheap and immediate. It records intent in the schema and commits - a new table version, with no data written. Declaring a column on a - billion-row table costs what declaring one on an empty table costs. -- Filling is a separate call you control, can retry, and -- against a deployment - -- can hand to the server as a [job](/basics/jobs) rather than waiting on. +| | Computed column | Materialized view | +| --- | --- | --- | +| Declaring or creating it | records the expression; no values are written | runs the query and stores its rows, unless created empty | +| A refresh | fills rows with no value and recomputes rows whose inputs changed | applies source changes or rebuilds the view, depending on the query and SDK version | +| A refresh when nothing changed | fills nothing | can be a no-op when incremental refresh is supported | +| In LanceDB Enterprise | runs as a [job](/basics/jobs) | runs as a server-side materialization job | -**Filling does not redo current work.** A refresh gives values to rows that -lack them and recomputes rows whose inputs changed since they were computed. It +### Computed columns fill incrementally {#incremental} + +A [computed column](/feature-engineering/add-computed-column) separates +**declaring** from **filling**: + +- Declaring is cheap and immediate. It records the expression in the schema + and commits a new table version, with no data written. It does not evaluate the expression over existing rows. +- Fill the column with a separate refresh call that you can retry. In LanceDB + Enterprise, submit the refresh as a [job](/basics/jobs) and track its progress. + +**Refresh new and changed rows.** A +[refresh](/feature-engineering/refresh-column) gives values to rows that lack +them and recomputes rows whose inputs changed since they were computed. It leaves the rest alone, so calling it twice is not twice the work, and appending rows means the next refresh handles only the new ones. -The corollary is worth stating plainly: **a filled value catches up with its -inputs only when you refresh.** If a column derives from `price` and a row's +**Computed values update when you refresh.** If a column derives from `price` and a row's `price` is later updated, the derived value keeps its old value until the next -refresh recomputes it. That is a deliberate trade -- it is what makes refreshes -idempotent and cheap -- but it will surprise you if you expect a spreadsheet. +refresh recomputes it. Schedule refreshes after changing the source data when consumers need +up-to-date features. -## Which one to reach for {#which-one} +### Materialized views store query results {#views} + +A [materialized view](/feature-engineering/materialized-views) has no separate +declaration step: creating one runs its query and stores the result, so the view +is ready to read. A refresh updates those stored results. Python SDK 0.40.0 can apply +changes incrementally for supported queries; other queries require a full rebuild. +See [refresh behavior](/feature-engineering/materialized-views#incremental) for +version-specific details. + +Enterprise runs materialization on remote compute. Use the SDK and refresh +options supported by your server version; see +[Enterprise refreshes](/feature-engineering/materialized-views#availability). + +## Choose a feature engineering workflow {#which-one} | | What it produces | Filled by | Available | | --- | --- | --- | --- | -| [Computed column](/feature-engineering/add-computed-column) | a new column on an existing table | a SQL expression | everywhere | -| [Materialized view](/feature-engineering/materialized-views) | a new table derived by a query | a refresh | everywhere | +| [Computed column](/feature-engineering/add-computed-column) | a new column on an existing table | a SQL expression | OSS and Enterprise | +| [Materialized view](/feature-engineering/materialized-views) | a new table derived by a query | a refresh | OSS and Enterprise | +| [Geneva](/geneva) | columns filled by Python | the `geneva` package | Enterprise | + +{/* +Draft for the Functions release. Replace the Geneva row of the table above with: | [Function](/management/function) | a computed column filled by Python | registered code | Enterprise | +| [Geneva](/geneva) | columns filled by Python, for existing pipelines | the `geneva` package | Enterprise | +*/} **Computed columns** are the common case: a column declared with a SQL expression, filled by [a refresh](/feature-engineering/refresh-column). @@ -57,23 +95,34 @@ expression, filled by [a refresh](/feature-engineering/refresh-column). aggregated view of a source -- and bring it up to date on each refresh. The result is an ordinary table: queryable, indexable, searchable. +**Geneva** is LanceDB Enterprise's Python package for work SQL cannot express: +embedding a column, calling a model, decoding an image. It defines features as +Python functions and backfills them as distributed jobs. + +{/* +Draft for the Functions release. Replaces the Geneva paragraph above: + **Functions** run Python where the data is, for work SQL cannot express: embedding a column, calling a model, decoding an image. Registered once, -versioned, then applied as a column. +versioned, then applied as a column. They are the way to do that on a +deployment; [Geneva](/geneva) remains available for existing pipelines. +*/} -They compose. A materialized view can select computed columns, and a computed -column can be filled by a registered function. +A materialized view can select computed columns. +{/* Draft for the Functions release: A computed column can be filled by a registered function. */} ## Where the work runs {#where} Enterprise -Locally, a refresh runs in your process. Against a deployment it runs across the +Locally, a refresh runs in your process. With LanceDB Enterprise it runs across the cluster, and the API reflects that: `refresh_column` raises, and -`refresh_column_async` returns a [job](/basics/jobs) handle instead. +`refresh_column_async` returns a [job](/basics/jobs) handle instead. Computed +columns are off by default in LanceDB Enterprise, and its administrator has to enable +them before a column can be declared there. ```python -job = table.refresh_column_async("embedding") +job = table.refresh_column_async("power_score") result = job.wait() result.rows_assigned ``` @@ -81,24 +130,28 @@ result.rows_assigned A backfill over a large table can therefore be started, left, and checked on from a different process -- the connection does not have to stay open. -That relocation is the main thing to understand about Enterprise feature -engineering. The API is the same; what changes is that the code runs somewhere -else, which is also why a function must declare the environment it needs. +Enterprise manages the compute for the refresh. You still control when to +submit it; declaring a computed column does not schedule recurring refreshes. +{/* Draft for the Functions release: That is also why a function must declare the environment it needs. */} ## Recovering from a failure {#failures} +{/* +Draft for the Functions release. + A function that raises fails the job. There is no per-row retry or skip policy in the function definition, so a function calling a flaky external service should handle that itself, inside the function body. +*/} -The alternative is to let the job fail, fix the cause, and run the refresh +When a refresh fails, fix the cause and run the refresh again. The new refresh fills what is still empty or stale. How much of the failed run was kept depends on where it ran (see [Refresh a column](/feature-engineering/refresh-column#failures)), and a retry can recompute values that were already written, so do not count on it costing only the rows that did not complete. -## Where to go next {#next} +## Related guides {#next} @@ -110,7 +163,10 @@ only the rows that did not complete. Bring a derived table up to date with its source. + {/* + Draft for the Functions release: Register Python to fill a column with. + */} diff --git a/docs/web/feature-engineering/materialized-views.mdx b/docs/web/feature-engineering/materialized-views.mdx index 7b00a6ce4..ab949adb1 100644 --- a/docs/web/feature-engineering/materialized-views.mdx +++ b/docs/web/feature-engineering/materialized-views.mdx @@ -1,9 +1,14 @@ --- title: "Refresh a materialized view" sidebarTitle: "Materialized View Refresh" -description: "A lakehouse object: a table defined by a query over another table, brought up to date by each refresh." +description: "Create a materialized view and refresh it when the source data changes." icon: table keywords: ["materialized view", "create_materialized_view", "refresh", "full refresh"] +availability: + oss: available + enterprise: available + summary: >- + Both offerings support materialized views. OSS refreshes run in your process; Enterprise runs the work on remote compute. See [Enterprise refreshes](#availability) for server compatibility requirements. --- import { @@ -11,12 +16,9 @@ import { PyMaterializedViewRefresh as MaterializedViewRefresh, } from '/snippets/materialized_view.mdx'; -A materialized view is a lakehouse object: a table whose contents are defined by -a query over another table. The query is recorded in the view's schema; -creating the view computes its rows, and a refresh brings them up to date. - -The result is an ordinary table. It can be queried, indexed, and searched, and -it appears in `table_names()` — nothing about reading it is special. +A materialized view stores the result of a query as a table. Use it to keep a +filtered or transformed dataset that you can query, index and search repeatedly. +Refresh the view after its source data changes. In SQL this is [`REFRESH MATERIALIZED VIEW`](/sql/refresh#refresh-materialized-view). @@ -36,89 +38,80 @@ reach the view through a refresh: {MaterializedViewRefresh} - -These results were verified with the development SDK 0.40.0b13, built from -lancedb main. The published 0.39.0 differs: the source must be created with -stable row ids (the `new_table_enable_stable_row_ids` storage option), -`create_materialized_view` creates the view empty and has no `with_no_data`, -and `refresh()` still accepts `full`. Its first `refresh()` writes every -matching row, two in this example. - +The example works with Python SDK 0.40.0. It creates the source with stable +row IDs, which that release requires for ordinary materialized-view sources. `select` takes column names, `(alias, expression)` pairs, or a dict of the same. Omitting it selects every source column, expanded against the schema as it was at creation time. `where` filters source rows; `limit` caps the view. -## Local sources no longer need stable row ids {#stable-row-ids} +## Prepare the source table {#stable-row-ids} -A local refresh does not track which source rows the view came from, so a local -view's source does not need stable row ids (`new_table_enable_stable_row_ids`): -the example above creates its source with the defaults. Earlier development -builds and the published 0.39.0 required them, and a deployment may still -require them; see [Against a deployment](#availability). +In Python SDK 0.40.0, create the source with +`storage_options={"new_table_enable_stable_row_ids": "true"}`, as the example +above does. Stable row IDs let the view track source rows across updates and +compaction. Set this option when creating the table; it cannot be added later. -The source is still checked, when the view is created and on every refresh: one -with an LSM write spec, or with retained write-ahead-log rows not yet compacted, -is refused. +A source with an LSM write specification or retained, uncompacted write-ahead-log +rows is not accepted by the local materialized-view implementation. -## Every local refresh rebuilds the view {#incremental} +## Refresh behavior {#incremental} -Locally, `refresh()` runs the view's query against the source and replaces -everything the view holds with the result, in one commit. It does not reconcile -changes: `rows_written` is the size of the whole result — two after adding Bea, -not one — and a refresh with nothing new to read still rewrites the view and -commits a new table version. +With Python SDK 0.40.0, a refresh applies source changes incrementally when the +query supports it, and rebuilds the view otherwise. In the example, adding Bea +writes one new view row. Refreshing again without source changes writes none. +Use `refresh(full=True)` to force a rebuild. -`source_version=N` refreshes against that source version instead of the latest, -which is how a view reproduces a known point in time. There is no `full` -argument; every local refresh is a full one. +`rows_written` counts rows written by that refresh, not all rows in the view. +Use `view.table.count_rows()` to get the total. Pass `source_version=N` to +refresh against a particular source version instead of the latest. -## Against a deployment {#availability} + +Refresh behavior has changed between SDK versions. The earlier 0.40.0b13 +development build rebuilt every local view on refresh and did not accept +`full`; 0.39.0 created views empty. Use the reference for your installed SDK +when maintaining an existing pipeline. + + +## Enterprise refreshes {#availability} Enterprise -`create_materialized_view` and `refresh` also work on a `db://` connection. -There the SDK sends them to the deployment as SQL -- `CREATE MATERIALIZED VIEW` -and `REFRESH MATERIALIZED VIEW` -- and the work runs on the cluster rather than -in your process. `source_version` is for local views only; the SDK refuses it -on a remote refresh. +Enterprise runs materialized-view creation and refresh on remote compute. +For SQL, see [`CREATE MATERIALIZED VIEW`](/sql/ddl/materialized-view) and +[`REFRESH MATERIALIZED VIEW`](/sql/dml/refresh#refresh-materialized-view). -Materialized views are off by default on a deployment and have to be enabled -there. How a deployment refreshes a view -- rebuilding the whole view, as above, -or reconciling what changed in the source -- depends on its LanceDB Enterprise -version and configuration, and so do the options -[`REFRESH MATERIALIZED VIEW`](/sql/dml/refresh#refresh-materialized-view) -accepts and whether the source table needs stable row ids. +Use an SDK version supported by your Enterprise deployment. Materialized-view +APIs and stored definitions have changed across releases; a local SDK example +alone does not establish compatibility with an older server. Before upgrading +an existing pipeline, confirm the supported refresh options and whether its +views need to be recreated with your administrator. -Stable row ids are set up on the deployment, not by the SDK: a `db://` -connection does not send `new_table_enable_stable_row_ids`, and no SDK call adds -them to an existing table. Depending on its version and configuration, a -deployment may give new tables stable row ids, may refuse a view created through -SQL -- as the SDK creates one -- over a source without them, or may convert an -existing source when a view is created through its REST API. Check with the -deployment's administrator what it requires before relying on a source. +Stable row IDs and feature configuration for Enterprise sources are managed on +the server. The source-table storage option in the local example does not +configure the Enterprise deployment, and Python SDK 0.40.0's `create_table` on a +`db://` connection does not accept `storage_options`: leave it out, and confirm +with your administrator that the deployment gives new tables stable row IDs. -`SHOW MATERIALIZED VIEWS` lists them, which is cheaper than -`list_materialized_views` -- that method finds views by reading every table's -schema, so it costs one open per table. +`list_materialized_views` requests the server's view listing. On a local +database it reads table schemas to find views. ## Working with the view {#working-with-the-view} `view.table` is the underlying table, for querying, indexing, or search. -`view.definition` is the recorded query — the select list, predicate, and -limit — which is how you check what a view actually contains without inferring -it from the rows. Both are properties; on an async view, the definition is read +`view.definition` is the recorded query: the select list, predicate, and +limit. Use it to inspect the query without inferring it from the rows. Both are properties; on an async view, the definition is read with `await view.definition()`. An existing view is reopened with `db.open_materialized_view(name)`. -## What to read next {#next} +## Related guides {#next} Derive a column instead of a whole table. - Every refresh commits a table version. + Inspect the table history and read a particular version. diff --git a/docs/web/feature-engineering/refresh-column.mdx b/docs/web/feature-engineering/refresh-column.mdx index d0f9f06a3..ae430e5f8 100644 --- a/docs/web/feature-engineering/refresh-column.mdx +++ b/docs/web/feature-engineering/refresh-column.mdx @@ -1,13 +1,19 @@ --- title: "Refresh a column" sidebarTitle: "Column Refresh" -description: "Fill the rows of a computed column that lack values — in process, or as a server-side job." +description: "Populate a computed column and update it when its inputs change." icon: arrows-rotate keywords: ["refresh_column", "backfill", "job", "idempotent", "REFRESH COLUMN"] +availability: + oss: available + enterprise: available + summary: >- + Both offerings support column refresh. The embedded library runs it in your process; Enterprise runs it as a server job that must be enabled for your Enterprise cluster. --- -A [declared column](/feature-engineering/add-computed-column) starts empty. A -refresh gives values to the rows that lack them. +Refresh a [computed column](/feature-engineering/add-computed-column) to fill +new rows and recompute values whose inputs changed. The example below continues +with the `doubled` column from the computed-column guide. ```python Python icon="python" @@ -37,8 +43,7 @@ The SQL form is [`REFRESH COLUMN`](/sql/refresh#refresh-column), with the same r A refresh fills rows that have no value and recomputes rows whose inputs changed since they were computed. Every other row is left alone, so a second -refresh with nothing changed does nothing. Two consequences follow, and both -are load-bearing. +refresh with nothing changed does nothing. This supports two common workflows. **Appending is cheap.** Rows added since the last refresh are filled by the next one; filled rows whose inputs are unchanged are skipped. Backfilling a @@ -63,16 +68,16 @@ REFRESH COLUMN embedding ON documents WHERE created_at > '2026-01-01' ``` -**Enterprise job type:** `computed_refresh` for a SQL-expression column, -`function_refresh` for one filled by a registered -[function](/management/function). +**Enterprise job type:** `computed_refresh` for a SQL-expression column. +{/* Draft for the Functions release: `function_refresh` for one filled by a registered [function](/management/function). */} -## As a job {#async} +## Run a refresh on Enterprise {#async} Enterprise -Against a deployment the refresh runs on the cluster. `refresh_column` raises +With an Enterprise cluster that enables computed columns, the refresh runs on the +cluster. `refresh_column` raises there; use `refresh_column_async`, which returns a [job](/basics/jobs) you can wait on or poll instead of holding a connection open: @@ -92,7 +97,7 @@ does not raise, which is what makes it safe in a loop; `wait()` raises A long backfill can therefore be started, left, and checked on from a different process. -### Resourcing the job {#resources} +### Configure workers and resources {#resources} SQL exposes the worker budget directly, which is the lever when a backfill is too slow or too expensive: @@ -111,7 +116,7 @@ rather than failing the job. What a failure keeps depends on where the refresh ran. `refresh_column` in your process writes the column in one commit, so a failure leaves the column as it -was. A refresh job on a deployment commits finished batches as it goes, so some +was. A refresh job in LanceDB Enterprise commits finished batches as it goes, so some values can remain after a failure. Either way, recover by fixing the cause and refreshing again. The new refresh @@ -121,12 +126,16 @@ that were already written -- for example when the record of the inputs they were computed from was not saved -- so do not count on a retry costing only the rows that did not complete. +{/* +Draft for the Functions release. + For a column filled by a [function](/management/function), the usual cause is the worker environment rather than the data. See [environment mismatches](/management/function#mismatch-symptoms) for the symptoms, which rarely name their cause. +*/} -## What to read next {#next} +## Related guides {#next} diff --git a/docs/web/governance/authorization.mdx b/docs/web/governance/authorization.mdx index 044db158e..9ef978c38 100644 --- a/docs/web/governance/authorization.mdx +++ b/docs/web/governance/authorization.mdx @@ -4,15 +4,13 @@ sidebarTitle: "Authorization" description: "Who may do what: privileges granted on catalog objects to principals, groups, roles, and API keys." icon: shield-halved keywords: ["authorization", "authz", "privileges", "rbac", "access control"] +availability: + oss: unavailable + enterprise: available + summary: >- + Enterprise supports catalog privileges when its administrator configures enforcement. Embedded LanceDB has no catalog authorization layer: access is whatever the object store allows. --- -Enterprise - -This page describes a LanceDB Enterprise capability. Embedded LanceDB has no -such layer -- access is whatever the object store enforces on the bucket or -directory, which is coarser and has no concept of a table. See -[Architecture](/architecture) for what differs between the two. - [Authentication](/enterprise/authentication) establishes who is calling. Authorization decides what they may do. @@ -28,15 +26,16 @@ A grant is held by one of four kinds of subject, distinguished by a prefix: | `p:` | a principal, by ID -- `P:` by name | | `g:` | a group, by ID -- `G:` by name | | `r:` | a role | -| `a:` | an API key | +| `a:` | the principal holding an API key, named by the key | Granting to a group or role rather than a principal is what keeps the entry count manageable: a principal's effective privileges are the union of what it holds directly and what it inherits. -Granting to an API key is narrower than granting to the principal that owns -it -- useful when a key is issued for one job and should not carry that -principal's full authority. +`P:`, `G:` and `a:` are ways of naming a subject when you make a request. The +entry is stored against the ID they resolve to, so a grant to `a:` belongs to +the principal holding that key and carries that principal's full authority, +not a narrower one. ## Objects {#objects} @@ -45,10 +44,28 @@ Objects are addressed by kind and path: ``` system database:db -namespace:db/ns -table:db/ns/table +namespace:db:ns +table:db:ns$table +view:db:ns$view ``` +{/* +Draft for the Functions release. Replaces the object kinds above: + +``` +system +database:db +namespace:db:ns +table:db:ns$table +view:db:ns$view +function:db:ns$function +secret:db:ns$secret +``` +*/} + +A nested namespace joins its path with `$`, as in `namespace:db:parent$child` +or `table:db:parent$child$table`. + The hierarchy is not decorative. Reaching a table requires `USAGE` on the namespace containing it and on the database containing that -- a grant on the table alone is not enough to traverse to it. @@ -57,10 +74,11 @@ table alone is not enough to traverse to it. | Privilege | Granted on | Allows | | --- | --- | --- | -| `OWNERSHIP` | any object | ownership, conveying full authority | +| `OWNERSHIP` | any object but `system` | ownership, conveying full authority | +| `USAGE` | system | listing jobs and their events, and looking up other identities | | `USAGE` | database | accessing the namespaces inside | | `USAGE` | namespace | accessing the tables inside | -| `SELECT` | table | reading rows | +| `SELECT` | table, view | reading rows | | `INSERT` | table | appending rows | | `UPDATE` | table | modifying existing rows | | `DELETE` | table | deleting rows | @@ -69,7 +87,19 @@ table alone is not enough to traverse to it. | `CREATE_TABLE` | namespace | creating a table in it | | `CREATE_MATERIALIZED_VIEW` | namespace | creating a materialized view in it | | `CREATE_VIEW` | namespace | creating a non-materialized view in it | -| `OPERATE` | system | reading server configuration | +| `OPERATE` | system | describing and controlling jobs, managing caches, and reading server configuration | + +{/* +Draft for the Functions release. Add these rows to the table above. +After the `USAGE` row for namespace: +| `USAGE` | function | binding it to a column, and refreshing a column it computes | +| `USAGE` | secret | binding and resolving it | +After the `UPDATE` row for table: +| `UPDATE` | secret | rotating it, without the right to bind or read it | +After the `CREATE_VIEW` row: +| `CREATE_FUNCTION` | namespace | registering a function in it | +| `CREATE_SECRET` | namespace | creating a secret in it | +*/} The four data privileges map onto the four [ingestion operations](/ingestion), so a principal that may append but not @@ -87,28 +117,32 @@ grant the other. A subtlety worth knowing, because it explains grants that appear to work on objects that do not exist yet. -When `USAGE` is checked on a database and **no entries exist for it at all**, -the database is assumed not to exist yet, and `CREATE_DATABASE` on `system` is -checked instead. The same applies to the default namespace: absent any -entries, the check falls back to `CREATE_NAMESPACE` on the containing -database. +When you create an object, the deployment checks the database and each +namespace that will contain it. One that **has no owner** -- because it does +not exist yet, or predates per-object authorization -- is treated as one you are +creating: the check falls back to `CREATE_DATABASE` on `system` for the +database, or to `CREATE_NAMESPACE` on its parent for a namespace, and you become +its owner. One that has an owner needs `USAGE` as usual. -So a principal holding `CREATE_DATABASE` can act on a database that has not -been created -- which is what makes creation possible at all, and why an empty -ACL set is not the same as a deny. +So a principal holding `CREATE_DATABASE` can create a table in a database that +has not been created -- which is what makes creation possible at all, and why +an empty ACL set is not the same as a deny. The fallback applies only when +creating; other operations on a database or namespace need `USAGE` on it. ## Listing grants {#listing} Entries can be listed filtered by object, by subject, or both: ``` -object = table:db/ns/documents +object = table:db:ns$documents subject = G:data-science ``` -Results paginate through a continuation token. One stored `(object, subject)` -tuple carrying several privileges expands into several entries in the -response, so the count reflects privileges rather than tuples. +Results paginate through a continuation token. A page holds at most 100 stored +`(object, subject)` tuples, and a tuple carrying several privileges expands +into several entries in the response, so a page can return more than 100 +entries: the count reflects privileges rather than tuples. Keep requesting with +the returned token until a page comes back without one. Listing by subject answers "what can this team reach"; listing by object answers "who can reach this table". Both are worth running before changing a diff --git a/docs/web/indexing/fts-index.mdx b/docs/web/indexing/fts-index.mdx index 4e18d5a86..832b67315 100644 --- a/docs/web/indexing/fts-index.mdx +++ b/docs/web/indexing/fts-index.mdx @@ -3,6 +3,13 @@ title: "Full-Text Search (FTS) Index" sidebarTitle: "FTS Index" description: "Create and tune BM25-based full-text search indexes in LanceDB." icon: "book" +availability: + oss: available + enterprise: available + summary: >- + Full-text indexes are built in the embedded library and in LanceDB Enterprise. + The embedded library builds the index before the call returns; a + deployment builds it in the background. --- import { PyFtsIndexAsync as FtsIndexAsync, PyFtsIndexCreate as FtsIndexCreate, PyFtsIndexNested as FtsIndexNested, PyFtsIndexWait as FtsIndexWait } from '/snippets/indexing.mdx'; @@ -114,7 +121,7 @@ await async_table.create_index("payload.text", config=FTS(with_position=True)) - `jieba/*` is for Chinese word segmentation with Jieba. - `lindera/*` loads a compiled Lindera dictionary, such as `lindera/ipadic` for Japanese or `lindera/ko-dic` for Korean. -Model-backed tokenizers such as `jieba/default`, `lindera/ipadic`, and `lindera/ko-dic` require tokenizer model files in Lance's language model home. Lance looks under the default platform data directory for `lance/language_models`, or you can set `LANCE_LANGUAGE_MODEL_HOME` to point to another model root. For example, `jieba/default` is resolved under `/jieba/default/...`. +Model-backed tokenizers such as `jieba/default`, `lindera/ipadic`, and `lindera/ko-dic` require tokenizer model files in Lance's language model home. Lance looks under the default platform data directory for `lance/language_models`, or you can set `LANCE_LANGUAGE_MODEL_HOME` to point to another model root. For example, `jieba/default` is resolved under `/jieba/default/...`. For a table in LanceDB Enterprise, the files must be present where the deployment builds and queries the index, so ask its administrator whether they are installed. `language` is used by token filters, not by the base tokenizer. Stemming supports Arabic, Danish, Dutch, English, Finnish, French, German, Greek, Hungarian, Italian, Norwegian, Portuguese, Romanian, Russian, Spanish, Swedish, Tamil, and Turkish. Built-in stop-word removal supports Danish, Dutch, English, Finnish, French, German, Hungarian, Italian, Norwegian, Portuguese, Russian, Spanish, and Swedish. For other stemming languages, set `remove_stop_words=False` or pass `custom_stop_words`. diff --git a/docs/web/indexing/index.mdx b/docs/web/indexing/index.mdx index b40fb7918..78844527f 100644 --- a/docs/web/indexing/index.mdx +++ b/docs/web/indexing/index.mdx @@ -3,20 +3,26 @@ title: "Indexing Data" sidebarTitle: "Overview" description: "Optimize search performance with LanceDB using vector indexes, full-text search, scalar indexes, and more." icon: "list" +availability: + oss: available + enterprise: available + summary: >- + Indexes are built in the embedded library and in LanceDB Enterprise. Enterprise builds them in the background, and builds fewer vector index types than the + embedded library. --- -An **index** is a data structure that facilitates efficient scans and lookups on an embedded dataset. -LanceDB provides a comprehensive suite of indexes to optimize performance across different use cases and data types: +Indexes speed up searches and filters by organizing data for specific query +patterns. LanceDB supports three main index families: - **Vector Index**: Efficiently searches for similar vectors across high-dimensional data (e.g. images, audio, or text embeddings) - **Full-Text Search Index**: Enables fast keyword-based searches by indexing words and phrases -- **Scalar Index**: Accelerates filtering and sorting of structured numeric or categorical data +- **Scalar Index**: Accelerates filtering of structured numeric or categorical data ## Supported Indexes | Index | Use Case | Description | | :--------- | :------- | :---------- | -| `IVF` (Vector) | Large-scale vector search with configurable accuracy/speed trade-offs. | Inverted File Index—a partition-based approximate nearest neighbor algorithm that groups similar vectors into partitions for efficient search.
**Quantizations**: `None/Flat` `PQ` `SQ` `RQ`| +| `IVF` (Vector) | Large-scale vector search with configurable accuracy/speed trade-offs. | Inverted File Index: a partition-based approximate nearest neighbor algorithm that groups similar vectors into partitions for efficient search.
**Quantizations**: `None/Flat` `PQ` `SQ` `RQ`| | `IVF_HNSW` (Vector) | Large-scale vector search requiring both high recall and efficient partitioning. Combines the scalability of IVF with the search quality of HNSW. | Hybrid index combining IVF partitioning with HNSW graphs in each partition. Provides improved search quality over pure IVF while maintaining scalability.
**Quantizations**: `None/Flat` `SQ` `PQ`| | `FTS` (Full-text search) | String columns (e.g., title, description, content) requiring keyword-based search with BM25 ranking. | Full-text search index using BM25 ranking algorithm. Tokenizes text with configurable tokenization, stemming, stop word removal, and language-specific processing. | | `BTree` (Scalar) | Numeric, temporal, and string columns with mostly distinct values. Best for selective equality, inequality, and range predicates. | Sorted index storing sorted copies of scalar columns with block headers in a btree cache. Header entries map to blocks of rows (4096 rows per block) for efficient disk reads. | @@ -24,6 +30,10 @@ LanceDB provides a comprehensive suite of indexes to optimize performance across | `LabelList` (Scalar) | List columns (e.g., tags, categories, keywords) requiring `array_contains_all` or `array_contains_any` filters. | Scalar index for `List` and `LargeList` columns of primitive values, using an underlying bitmap index structure to enable fast array membership lookups. | | `FM` (Scalar) | String or binary columns that need raw substring search. | FM-Index over `Utf8`, `LargeUtf8`, `Binary`, or `LargeBinary` data for filters such as `contains(path, 'needle')`. Use FTS instead for tokenized word search and BM25 ranking. | +A deployment builds the vector indexes `IVF_FLAT`, `IVF_PQ`, `IVF_RQ` and +`IVF_HNSW_SQ`. `IVF_SQ`, `IVF_HNSW_FLAT` and `IVF_HNSW_PQ` are built only by the +embedded library. + ## Quantization LanceDB also supports several [quantization](/indexing/vector-index) methods, used by vector indexes to compress vectors and reduce storage requirements: @@ -32,26 +42,30 @@ LanceDB also supports several [quantization](/indexing/vector-index) methods, us | `PQ` (Product Quantization) | Default choice for most vector search scenarios. Use when you need to balance index size and recall. | Divides vectors into subvectors and quantizes each subvector independently. Provides a good balance between compression ratio and search accuracy. | | `RQ` (RabitQ Quantization) | Use when you need maximum compression or have specific per-dimension requirements. | Per-dimension quantization using a RabitQ codebook. Provides fine-grained control over compression per dimension. For `IVF_RQ`, vector dimensions must be divisible by `8`. | | `SQ` (Scalar Quantization) | Use when you need faster indexing or when vector dimensions have consistent value ranges. | Quantizes each dimension independently. Simpler than PQ but typically provides less compression. | -| `None/Flat` | Use for binary vectors (with `hamming` distance) or when you need maximum recall and have sufficient storage. | No quantization—stores raw vectors. Provides the highest accuracy but requires more storage and memory. | +| `None/Flat` | Use for binary vectors (with `hamming` distance) or when you need maximum recall and have sufficient storage. | No quantization; stores raw vectors. Provides the highest accuracy but requires more storage and memory. | ## Performance {#performance} #### Vector indexes {#vector-indexes} -If vector search latency climbs with table size (i.e., queries that ran in milliseconds on a small table take seconds as it grows to millions of rows), the cause is the default brute-force scan over every vector. That works fine below ~100K vectors, but past that you should build a dedicated vector index. Pick the type by your data shape: +If vector search latency climbs with table size (i.e., queries that ran in milliseconds on a small table take seconds as it grows to millions of rows), check whether the query is scanning every vector without an index. The point at which an index helps depends on vector dimensions, hardware and +your latency target. Measure your workload before choosing an index. Pick the type by your data shape: | Index | When to use | |-------|-------------| -| `IVF_PQ` | General-purpose default; what Enterprise builds automatically. Higher accuracy than RQ at small dimensions (≤ 256). | +| `IVF_PQ` | General-purpose default, and what a deployment builds when it is set to index a column automatically. Higher accuracy than RQ at small dimensions (≤ 256). | | `IVF_RQ` | Maximum compression on high-dim vectors, faster builds than PQ. | | `IVF_HNSW_SQ` | Best recall/latency for unfiltered search; higher latency variance under selective filters. | | `IVF_FLAT` | Required for binary vectors with `hamming`. | -The distance metric is fixed once the index is built. Pick the distance metric based on how the embedding model was trained: `cosine` (unnormalized), `dot` (already-normalized, best performance), `l2` (general-purpose, default), `hamming` (binary). If a query explicitly requests a different metric, the index cannot be used and the query falls back to a brute-force scan; LanceDB Enterprise may reject that scan on a large table. For parameter tuning, see [Vector Indexing](/indexing/vector-index). +The distance metric is fixed once the index is built. Pick the distance metric based on how the embedding model was trained: `cosine` (unnormalized), `dot` (already-normalized, best performance), `l2` (general-purpose, default), `hamming` (binary). If a query explicitly requests a different metric, the index cannot be used and the query falls back to a brute-force scan; a deployment can be configured to reject that scan on a large table. For parameter tuning, see [Vector Indexing](/indexing/vector-index). #### Scalar indexes {#scalar-indexes} -If filtered queries slow down as the table grows — even when the filter is selective — the cause is a full column scan: without a scalar index, LanceDB evaluates the `where(...)` predicate on every row, and the same applies to `merge_insert` join keys. The best practice is to build a scalar index on every column you filter or join on, picking the type by the column's shape: +Scalar indexes can reduce the rows scanned for selective filters and speed up +`merge_insert` key lookups. Start with columns used frequently in those operations, +then use [query plans](/search/explain) to check whether the index helps. Choose +the index type based on the column's values: | Index | Best for | |-------|----------| @@ -63,4 +77,6 @@ See [Scalar Indexing](/indexing/scalar-index). #### Full-text search {#full-text-search} -If your full-text index is much larger than expected, or takes longer than expected to build, the cause is usually phrase-query flags being enabled when they aren't needed: `with_position=True` and `remove_stop_words=False` both significantly inflate index size and build time. The best practice is to keep the defaults for most workloads, and only enable those flags when you actually need to search for phrases. See [FTS Indexing](/indexing/fts-index) configuration options for the full set of options. +Token positions and retained stop words increase the information stored in a +full-text index. Keep the defaults for basic keyword search; enable +`with_position=True` and retain the needed stop words for phrase queries. See [FTS Indexing](/indexing/fts-index) configuration options for the full set of options. diff --git a/docs/web/indexing/reindexing.mdx b/docs/web/indexing/reindexing.mdx index dd50f62fe..fdbfe3b5f 100644 --- a/docs/web/indexing/reindexing.mdx +++ b/docs/web/indexing/reindexing.mdx @@ -4,6 +4,14 @@ sidebarTitle: "Build & Repair" description: "Learn how to keep your indexes up-to-date in LanceDB using incremental indexing, including best practices for adding new records without full reindexing." icon: "refresh" keywords: ["reindexing", "re-index", "incremental indexing", "reindex"] +availability: + oss: available + enterprise: available + summary: >- + Indexes are kept up to date in the embedded library and in LanceDB Enterprise. In + the embedded library you add new rows to them with `optimize()`; a + deployment does it in the background, and `optimize()` is not available + there. --- import { @@ -20,16 +28,19 @@ As data is being added and a reindex operation is running, LanceDB will combine Rather than dropping an existing index entirely and reindexing from scratch, LanceDB supports **incremental indexing**. -**Enterprise job types:** building an index runs as `create_ivf_pq`, `create_btree`, -`create_fts`, and one more per index algorithm -- the type names the algorithm, -not a separate feature. Folding new rows into an existing one runs as the -matching `reindex_*` type instead. +**Enterprise job types:** an index build runs as `reindex_` followed by the +algorithm, such as `reindex_ivf_pq`, `reindex_btree` or `reindex_fts`, and so +does folding new rows into an existing index. A create-only build, one that must +not replace an existing index, runs as `create_` followed by the algorithm +instead; Python's `create_scalar_index` and `create_fts_index` ask for that by +default. The name says which algorithm is building, not a separate feature. ## Incremental Reindexing {#incremental-reindexing} -You can manually trigger an incremental indexing operation on updated data -using the `optimize()` method on a table. +In the embedded library, you trigger an incremental indexing operation on +updated data with the `optimize()` method on a table. A deployment does this +for you and does not support `optimize()`, as described below. Table optimization performs three maintenance operations: @@ -45,10 +56,12 @@ Table optimization performs three maintenance operations: Enterprise -LanceDB Enterprise support incremental reindexing through an automated background process. When new data is added to a table, the system automatically triggers a new index build. As the dataset grows, indexes are asynchronously updated in the background. +LanceDB Enterprise supports incremental reindexing through an automated background process. Once enough new rows have been added to a table, the system automatically adds them to its indexes. As the dataset grows, indexes are asynchronously updated in the background. -- While indexes are being rebuilt, queries use brute force methods on unindexed rows, which may temporarily increase latency. To avoid this, set `fast_search=True` to search only indexed data. +- While indexes are being rebuilt, queries use brute force methods on unindexed rows, which may temporarily increase latency. If you can omit unindexed rows, set `fast_search=True` to search only indexed data. + This can return incomplete results while indexing catches up. - Use `index_stats()` to view the number of unindexed rows. This will be zero when indexes are fully up-to-date. If you call `wait_for_index(...)`, it polls the same status and can time out while continuous writes keep adding unindexed rows. +- Rows below the deployment's threshold stay unindexed until more arrive, so the count can stay above zero without further writes. `FM` indexes are not updated in the background. The benefit of using LanceDB Enterprise is that it automates the reindexing process and operates continuously in the background, minimizing the impact on latency under high loads. @@ -71,9 +84,9 @@ If you need to reclaim space more aggressively in OSS, use a shorter retention w ## Building on GPUs {#gpu} -With LanceDB's GPU-powered vector indexing you can index very large datasets in far less time -than you could with the default CPU-based indexing. In our tests, LanceDB -is capable of indexing billions of rows in under four hours on a 1-8 GPU cluster. +GPU acceleration can reduce the time needed to build vector indexes on large +datasets. Actual build time depends on the index configuration, vector +dimensions, storage throughput and available GPU resources. **Automatic GPU indexing** @@ -82,9 +95,8 @@ is capable of indexing billions of rows in under four hours on a 1-8 GPU cluster Automatic GPU Indexing is currently only available in [LanceDB Enterprise](/enterprise/). Please [contact us](https://www.lancedb.com/contact) to enable this feature for your deployment. -The vector index is created when you call `create_index`. The backend will use GPU resources -to build either the IVF or HNSW indexes. The system automatically selects the optimal GPU -configuration based on your data size and available hardware. +The vector index is created when you call `create_index`. The backend uses GPU resources +to build `IVF_PQ` indexes, with the number of GPUs set by the deployment's configuration. This process is also asynchronous by default, but you can use `wait_for_index` to convert it into a synchronous process by waiting until the index is built. @@ -93,9 +105,9 @@ into a synchronous process by waiting until the index is built. `wait_for_index(...)` waits for the index to exist and for `index_stats(...)` to report `num_unindexed_rows == 0`. It can time out if the table is receiving continuous writes while the build is trying to catch up. -GPU acceleration changes how the vector index is built, not the lifecycle after the build. Rows -appended later still need `optimize()` before they are part of the index, and `fast_search()` only -searches indexed rows. +GPU acceleration changes how the vector index is built, not the lifecycle after the build. In the +embedded library, rows appended later still need `optimize()` before they are part of the index; a +deployment adds them in the background. `fast_search()` only searches indexed rows. ### Manual GPU indexing in LanceDB OSS {#gpu-manual-gpu-indexing-in-lancedb-oss} @@ -125,9 +137,9 @@ to enable GPU training on your device. ### Performance considerations {#gpu-performance-considerations} - GPU memory usage scales with `num_partitions` and vector dimensions -- For optimal performance, ensure GPU memory exceeds dataset size -- Batch size is automatically tuned based on available GPU memory -- Indexing speed improves with larger batch sizes +- Size GPU memory for the training batches and index configuration +- Monitor memory use while choosing training batch sizes +- Measure build time and recall before changing training settings ### Troubleshooting {#gpu-troubleshooting} diff --git a/docs/web/indexing/scalar-index.mdx b/docs/web/indexing/scalar-index.mdx index e8dcf6bb7..9f1f9a176 100644 --- a/docs/web/indexing/scalar-index.mdx +++ b/docs/web/indexing/scalar-index.mdx @@ -3,6 +3,13 @@ title: "Scalar Indexes" sidebarTitle: "Scalar Index" description: "Learn how to use scalar indexes in LanceDB for efficient metadata filtering and query optimization." icon: "tree" +availability: + oss: available + enterprise: available + summary: >- + Scalar indexes are built in the embedded library and in LanceDB Enterprise. A + deployment builds them in the background and keeps most of them up to date + itself; in the embedded library you update them with `optimize()`. --- import { @@ -19,7 +26,9 @@ import { PyScalarIndexNestedFields as ScalarIndexNestedFields, } from '/snippets/indexing.mdx'; -Scalar indexes organize data by scalar attributes (e.g., numbers, categories) and enable fast filtering of vector data. They accelerate retrieval of scalar data associated with vectors, thus enhancing query performance. +Scalar indexes accelerate filters on values such as numbers, categories and +strings. Use them for selective table queries or metadata filters combined +with vector and full-text search. LanceDB supports four types of scalar indexes: @@ -29,7 +38,7 @@ LanceDB supports four types of scalar indexes: - `FM`: FM-Index over string or binary columns that accelerates substring search via `contains(col, 'needle')`. -Scalar indices serve as a foundational optimization layer, accelerating filtering across diverse search workloads. They can be combined with: +Use scalar indexes with: - Vector search (prefilter results using metadata) - Full-text search (combining keyword matching with structured filters) @@ -70,11 +79,11 @@ If you are using LanceDB Enterprise, the `create_scalar_index` API returns immed -`wait_for_index(...)` waits until the named scalar indexes exist and `index_stats(...)` reports `num_unindexed_rows == 0`. If a table is receiving steady writes, that fully indexed state may not stabilize before the timeout. +`wait_for_index(...)` waits until the named scalar indexes exist and `index_stats(...)` reports `num_unindexed_rows == 0`. If a table is receiving steady writes, that fully indexed state may not stabilize before the timeout. In LanceDB Enterprise, rows added after the index was built are indexed only once enough have accumulated, so a small remainder can also keep the wait from finishing. ### 3. Update the Index -Updating the table data (adding, deleting, or modifying records) requires that you also update the scalar index. This can be done by calling `optimize`, which will trigger an update to the existing scalar index. +Updating the table data (adding, deleting, or modifying records) requires that you also update the scalar index. In the embedded library, call `optimize`, which will trigger an update to the existing scalar index. @@ -83,7 +92,7 @@ Updating the table data (adding, deleting, or modifying records) requires that y -New data added after creating the scalar index will still appear in search results if optimize is not used, but with increased latency due to a flat search on the unindexed portion. LanceDB Enterprise automates the optimize process, minimizing the impact on search speed. +New data added after creating the scalar index will still appear in search results if optimize is not used, but with increased latency due to a flat search on the unindexed portion. LanceDB Enterprise does not support `optimize()`; instead it adds new rows to `BTREE`, `BITMAP` and `LABEL_LIST` indexes in the background once enough have accumulated. It does not update `FM` indexes automatically. ### 4. Run Indexed Searches @@ -125,7 +134,7 @@ Nested paths follow Lance field-path semantics: dot-separate each struct field f The `FM` index is a scalar index built over string or binary columns that accelerates substring lookups expressed as `contains(col, 'needle')`. Unlike the tokenized [FTS index](/indexing/fts-index), which matches whole words after -tokenization, the FM-Index matches arbitrary substrings of the raw bytes — so it +tokenization, the FM-Index matches arbitrary substrings of the raw bytes. It works well for URLs, file paths, identifiers, log lines, or any column where you search for a fragment rather than a word. @@ -143,6 +152,8 @@ or language-aware tokenization. Build an FM-Index with the async `create_index` API by passing the `Fm` config in Python or `Index.fm()` in TypeScript. In Rust, use `Index::Fm(FmIndexBuilder::default())`. +A deployment builds FM-Indexes too, but does not add new rows to them in the background as it +does for the other scalar indexes. ```python Python icon="python" diff --git a/docs/web/indexing/vector-index.mdx b/docs/web/indexing/vector-index.mdx index b10da3f6f..e9768efa6 100644 --- a/docs/web/indexing/vector-index.mdx +++ b/docs/web/indexing/vector-index.mdx @@ -3,6 +3,14 @@ title: "Vector Indexes" sidebarTitle: "Vector Index" description: "Build and manage LanceDB vector indexes." icon: "arrow-up-right-dots" +availability: + oss: available + enterprise: available + summary: >- + Vector indexes are built in the embedded library and in LanceDB Enterprise. The + embedded library builds an index before `create_index()` returns; a + deployment builds it in the background, keeps it up to date, and builds + only `IVF_FLAT`, `IVF_PQ`, `IVF_RQ` and `IVF_HNSW_SQ`. --- import { @@ -53,7 +61,7 @@ Lance with `index_type`. | Maximum compression | `IVF_RQ` | IVF indexing with RaBitQ quantization | Around `1/32` of raw size | `IvfRq` | | Higher accuracy at small dimensions (`dimension <= 256`) | `IVF_PQ` | IVF indexing with product quantization | Usually `1/64` to `1/16` of raw size (depends on `num_sub_vectors`) | `IvfPq` | | Best recall/latency trade-off | `IVF_HNSW_SQ` | IVF-HNSW indexing with scalar | Typically a little larger than `1/4` of raw size | `IvfHnswSq` | -| Highest recall / no quantization | `IVF_HNSW_FLAT` | IVF-HNSW indexing with no quantization | Around raw vector size plus HNSW graph overhead | `IvfHnswFlat` | +| Highest recall / no quantization | `IVF_HNSW_FLAT` | IVF-HNSW indexing with no quantization; embedded library only | Around raw vector size plus HNSW graph overhead | `IvfHnswFlat` | If your vector search frequently includes metadata filters (`where(...)`), use `IVF_RQ` or `IVF_PQ`. In filtered workloads, HNSW-backed IVF indexes such as `IVF_HNSW_FLAT` and `IVF_HNSW_SQ` can show higher latency variance. @@ -114,7 +122,7 @@ your destination. This iterative series of road networks mimics the layered grap In LanceDB, HNSW is not exposed as a top-level vector index. Instead, it's available as a substructure which further indexes the selected vectors inside each IVF partition. This combines the scalability of IVF with the high recall of HNSW. -LanceDB supports IVF-HNSW-based quantized indexes `IVF_HNSW_FLAT`, `IVF_HNSW_PQ`, and `IVF_HNSW_SQ`. +LanceDB supports IVF-HNSW-based quantized indexes `IVF_HNSW_FLAT`, `IVF_HNSW_PQ`, and `IVF_HNSW_SQ`; a deployment builds only `IVF_HNSW_SQ`. @@ -141,7 +149,7 @@ Learn how to configure, build, and search LanceDB vector indexes, including buil | `limit` | Number of results to return (the `k` in `k-ANN`). | | `nprobes` | Sets both `minimum_nprobes` and `maximum_nprobes` to the same value in Lance. Leave unset to use Lance's adaptive defaults. At very high scales you will likely need to experiment to find your preferred recall / latency balance. | | `minimum_nprobes` | Minimum number of partitions scanned. Leave unset to use Lance's default. | -| `maximum_nprobes` | Maximum number of partitions scanned. Only scans more than `minimum_nprobes` if an initial pass does not return enough results — useful for narrow filters. Leave unset, or set to `0`, to remove the cap. | +| `maximum_nprobes` | Maximum number of partitions scanned. Only scans more than `minimum_nprobes` if an initial pass does not return enough results, which is useful for narrow filters. Leave unset, or set to `0`, to remove the cap. | | `ef` | HNSW search-time exploration factor. Start around `1.5 * k` and increase up to `10 * k` for higher recall. | | `refine_factor` | Reads and reranks additional candidates in memory to recover recall lost to quantization. Start between `5` and `50` (a factor of `25` reads and reranks the top `25 * k` candidates) and measure the latency cost, typically a few milliseconds, against the recall gained. | | `distance_range(lower_bound, upper_bound)` | Return only rows whose distance falls within `[lower_bound, upper_bound)`. Either bound is optional. Useful for near-duplicate detection or "close-enough" matching. | @@ -152,8 +160,8 @@ The `distance_type` can be specified at search time in addition to build time. I you do this, set it to the same value used to build the vector index. If the search metric does not match the index metric, LanceDB cannot use the index and falls back to a costly flat search. On LanceDB Cloud, the brute-force KNN safety limit -can reject that search on a large table instead of executing it. LanceDB -Enterprise doesn't have the limit. +can reject that search on a large table instead of executing it. An Enterprise +deployment applies the same limit only when it is configured to. When the search metric is omitted, current SDKs use the index metric if an index is available. Some older SDK versions sent `l2` when it was omitted, so explicitly @@ -173,6 +181,8 @@ older client. **Filtered queries and adaptive `nprobes`.** When a `where(...)` filter is active, LanceDB initially scans `minimum_nprobes` partitions and uses a wider scan if too few rows are found. Set `minimum_nprobes == maximum_nprobes` or call `nprobes(n)` to instead fix the partition count. +Whether a deployment widens the scan this way depends on its version; one that does not scans exactly +`minimum_nprobes` partitions. Here is an example of a vector search exercising several of the above parameters. @@ -211,7 +221,7 @@ Multivector indexing currently requires `distance_type="cosine"`. Use `bypass_ve #### Async API and Config Objects Create vector indexes asynchronously with `await table.create_index("vector", config=...)`. The `config` object -admits the same build-time parameters described above — pass an instance of a Python config class from the table in [Choosing the Right Index](#choosing-the-right-index): +admits the same build-time parameters described above. Pass an instance of a Python config class from the table in [Choosing the Right Index](#choosing-the-right-index): @@ -234,7 +244,7 @@ Specify `vector_column_name` if you have multiple vector columns or non-default TypeScript currently doesn't support `IvfSq` (IVF with Scalar Quantization). -For a vector field nested inside a struct, pass its full dotted path as `vector_column_name` (e.g. `image.embedding`) — the same path is used at query time and is what `list_indices()` reports under `columns`. See [Selecting the vector column](/search/vector-search#selecting-the-vector-column) for the full path syntax. +For a vector field nested inside a struct, pass its full dotted path as `vector_column_name` (e.g. `image.embedding`). The same path is used at query time and is what `list_indices()` reports under `columns`. See [Selecting the vector column](/search/vector-search#selecting-the-vector-column) for the full path syntax. @@ -265,17 +275,18 @@ The snippet below builds and queries an `IVF_HNSW_SQ` index. Enterprise-only In LanceDB Enterprise, vector indexes are managed **automatically**. The system asynchronously updates and optimizes indexes as a background process: - Automatically manages indexing parameters and storage -- Infers vector columns from the schema -- GPU-based indexing is available on request -- [contact support](/deployment/support) to enable it for your deployment. A few billion vectors index in under 4 hours on GPU. +- Can build an `IVF_PQ` index automatically on a vector column the deployment is configured to index +- GPU-based `IVF_PQ` builds are available on request -- [contact support](/deployment/support) to enable them for your deployment. Build time depends on dataset size, dimensions and GPU resources. Open-Source LanceDB OSS users can manually create vector indexes by calling `table.create_index()`. See the above sections for guidance on manually tuning index parameters as data changes. -`create_index()` returns immediately, but the vector index builds asynchronously. -To wait until all data is indexed, specify the `wait_timeout` parameter, or call `wait_for_index(...)` afterward — -it waits for the named index to exist and for `index_stats(...)` to report `num_unindexed_rows == 0`. +In LanceDB Enterprise, `create_index()` returns immediately and the vector index builds asynchronously. +To wait until all data is indexed, specify the `wait_timeout` parameter, or call `wait_for_index(...)` afterward. +It waits for the named index to exist and for `index_stats(...)` to report `num_unindexed_rows == 0`. +In the embedded library the index is built before `create_index()` returns. @@ -287,11 +298,12 @@ fallback and searches only indexed rows. **Operational checks** -After appends or other writes, use `optimize()` to fold new rows into existing indexes. +In the embedded library, use `optimize()` after appends or other writes to fold new rows into existing indexes. +A deployment does this in the background, and `optimize()` is not available there. #### Check Index Status -Vector index creation runs in the background and may take some time to complete. +In LanceDB Enterprise, vector index creation runs in the background and may take some time to complete. While it is ongoing, you can check its status through the API or the **LanceDB Enterprise UI**. In the LanceDB Enterprise UI, navigate to your table page - the "Index" column reflects each column's index status: it is blank when no index exists, shows an "in progress" label while the index is being built, and shows the index type once the build completes. @@ -318,7 +330,7 @@ Each entry returned by `list_indices()` also carries detailed per-index metadata -These fields are populated for local and embedded tables. On LanceDB Enterprise remote tables they are returned as `None` / `undefined` until the server response surfaces them. +These fields are populated for local tables. In LanceDB Enterprise, `list_indices()` returns them except `num_unindexed_rows`; use `index_stats()` for index coverage. #### Custom Index Names @@ -343,9 +355,10 @@ compactly store an index. #### Product Quantization (PQ) To visualize PQ, assume a vector dataset has $d$ dimensions, with a **chunk** of a vector denoting a contiguous block of entries. Imagine that each vector has - $m$ disjoint chunks of $d/m$ entries each, where the first chunk represents entries $0$ through $d-1$, the second contains entries $d$ through $2d - 1$, and so on. + $m$ disjoint chunks of $d/m$ entries each, where the first chunk contains entries $0$ through $d/m-1$, the second contains +entries $d/m$ through $2d/m-1$, and so on. -Now, for each $i$, let $S_i$ represent the set containing chunk $i$ of each vector (entries $(i-1)d$ to $di - 1$). +Now, for each $i$, let $S_i$ represent the set containing chunk $i$ of each vector (entries $(i-1)d/m$ to $id/m-1$). For each $S_i$ independently, a small set of **centroids** is computed corresponding to an approximate solution to the $k$-means clustering problem. For each original vector in the dataset, every chunk is associated to its nearest centroid; thus, the vector itself is associated with the concatenation of $m$ centroids. @@ -397,7 +410,7 @@ among all integral multiples $\{j \times 10^5: -128 \leq j \leq 127 \}$. | Parameter | Description | | :--- | :--- | -| `num_bits` | Bits per dimension for quantization. Only applies to `IVF_PQ`/`IVF_HNSW_PQ` (default `8`) and `IVF_RQ` (default `1`, RaBitQ) — not used by `IVF_FLAT`/`IVF_SQ`/`IVF_HNSW_FLAT`/`IVF_HNSW_SQ`. Higher values improve accuracy at the cost of additional storage. | +| `num_bits` | Bits per dimension for quantization. Only applies to `IVF_PQ`/`IVF_HNSW_PQ` (default `8`) and `IVF_RQ` (default `1`, RaBitQ); not used by `IVF_FLAT`/`IVF_SQ`/`IVF_HNSW_FLAT`/`IVF_HNSW_SQ`. Higher values improve accuracy at the cost of additional storage. | `max_iterations` and `sample_rate` also affect quantizer training, but since they apply to every IVF/HNSW index type (not just quantized ones), they're documented as general [Build-time Parameters](/indexing/vector-index#build-time-parameters) instead. All three are passed as keyword arguments to `create_index`, alongside `index_type`: diff --git a/docs/web/ingestion/create-table.mdx b/docs/web/ingestion/create-table.mdx index fce23ac65..7d4363be7 100644 --- a/docs/web/ingestion/create-table.mdx +++ b/docs/web/ingestion/create-table.mdx @@ -1,9 +1,14 @@ --- title: "Creating a table" sidebarTitle: "Create Table" -description: "Start here: turn the data you already have into a LanceDB table." +description: "Create a table from records or a schema and choose input formats." icon: table keywords: ["create table", "ingest", "arrow", "pandas", "polars", "pydantic", "schema"] +availability: + oss: available + enterprise: available + summary: >- + Create and open tables in OSS and Enterprise. --- import { @@ -11,65 +16,60 @@ import { RsConnect, } from '/snippets/connection.mdx'; import { - PyCreateTableFromDicts as CreateTableFromDicts, TsCreateTableFromDicts as TsCreateTableFromDicts, RsCreateTableFromDicts as RsCreateTableFromDicts, - PyCreateTableConflictHandling as CreateTableConflictHandling, TsCreateTableConflictHandling as TsCreateTableConflictHandling, RsCreateTableConflictHandling as RsCreateTableConflictHandling, - PyCreateTableFromPandas as CreateTableFromPandas, - PyCreateTableCustomSchema as CreateTableCustomSchema, TsCreateTableCustomSchema as TsCreateTableCustomSchema, RsCreateTableCustomSchema as RsCreateTableCustomSchema, - PyCreateTableFromPolars as CreateTableFromPolars, - PyCreateTableFromArrow as CreateTableFromArrow, TsCreateTableFromArrow as TsCreateTableFromArrow, RsCreateTableFromArrow as RsCreateTableFromArrow, - PyCreateTableFromPydantic as CreateTableFromPydantic, - PyCreateTableNestedSchema as CreateTableNestedSchema, - PyOpenExistingTable as OpenExistingTable, TsOpenExistingTable as TsOpenExistingTable, RsOpenExistingTable as RsOpenExistingTable, - PyCreateEmptyTable as CreateEmptyTable, TsCreateEmptyTable as TsCreateEmptyTable, RsCreateEmptyTable as RsCreateEmptyTable, - PyCreateEmptyTablePydantic as CreateEmptyTablePydantic, - PyDropTable as DropTable, TsDropTable as TsDropTable, RsDropTable as RsDropTable, PyTablesBasicConnect as TablesBasicConnect, PyTablesDocumentModel as TablesDocumentModel, PyTablesTzValidator as TablesTzValidator, } from '/snippets/tables.mdx'; +import { + PyCreateTableFromDicts as CreateTableFromDicts, + PyCreateTableConflictHandling as CreateTableConflictHandling, + PyCreateTableFromPandas as CreateTableFromPandas, + PyCreateTableCustomSchema as CreateTableCustomSchema, + PyCreateTableFromPolars as CreateTableFromPolars, + PyCreateTableFromArrow as CreateTableFromArrow, + PyCreateTableFromPydantic as CreateTableFromPydantic, + PyCreateTableNestedSchema as CreateTableNestedSchema, + PyOpenExistingTable as OpenExistingTable, + PyCreateEmptyTable as CreateEmptyTable, + PyCreateEmptyTablePydantic as CreateEmptyTablePydantic, + PyDropTable as DropTable, +} from '/snippets/ingestion_examples.mdx'; -A table is created from data you already have -- a list of objects, an Arrow -table, a pandas or polars frame, Pydantic models -- or from a schema alone when -the rows come later. +import { PyWriteWorkflow, TsWriteWorkflowBody, TsWriteWorkflowTail, RsWriteWorkflow } from '/snippets/ingestion_examples.mdx'; + +Create a table from existing records, or define an empty table before data +arrives. Define a schema to specify column types, allow or disallow missing +values, and set the size of your vectors. ## Create a table with data {#create-a-table-with-data} -Initialize a LanceDB connection and create a table +The Python input examples below are complete programs. Install `lancedb` and +any additional libraries imported by the example, save one as a Python file, +and run it. Each uses its own temporary local database. - - - {TablesBasicConnect} - - - - {TsConnect} - - - - {RsConnect} - - - -Depending on the SDK, LanceDB can ingest arrays of records, Arrow tables or record batches, and Arrow batch iterators or readers. Let's take a look at some of the common patterns. +The TypeScript and Rust blocks illustrate the corresponding input formats. +For a complete connection-to-result program in either language, expand +[the write workflow](#complete-example). Table names follow the +[naming rules](/basics/naming). ### From list of objects {#from-list-of-objects} You can provide a list of objects to create a table. The Python and TypeScript SDKs -support lists/arrays of dictionaries, while the Rust SDK supports lists of structs. +support lists/arrays of records. The Rust example converts structs to Arrow batches. In Python, pass a list or other batch-like object; a single bare `dict` or single `LanceModel` is rejected. @@ -87,37 +87,42 @@ In Python, pass a list or other batch-like object; a single bare `dict` or singl -### Handle existing tables {#handle-existing-tables} +### If the table already exists {#handle-existing-tables} -By default, `create_table` raises an error if a table with the same name already exists. -You can change this behavior with two parameters that resolve the conflict in different ways: +By default, creating a table fails if its name is already in use. Choose whether +to open that table or overwrite it. -- **Idempotent open**: return the existing table without modifying it. Use when your - code may run more than once (notebooks, retries, init scripts) and you want to reuse - the table on subsequent runs. The provided data is ignored, but the schema is - validated against the existing table and a mismatch raises an error. -- **Overwrite**: drop the existing table and create a new one with the provided data. - Use this for test fixtures or when you intentionally want to replace prior contents. - This permanently discards the old table's data. +#### Open the existing table + +Use `exist_ok=True` in Python or `existOk: true` in TypeScript to return the +existing table without changing its contents. This is useful for setup code +that may run more than once. Any data supplied to the create call is ignored +when the table already exists. + +Inspect the returned table's schema if your code depends on specific columns +or types. Opening an existing table does not change its schema to match the +data supplied in the create call. + +#### Overwrite the table + +Use `mode="overwrite"` in Python or `mode: "overwrite"` in TypeScript to replace +the existing table with the supplied data and schema. Choose this only when +you intend to replace its previous contents. + +The following Python program demonstrates both choices: - {CreateTableConflictHandling} - - {TsCreateTableConflictHandling} - - - {RsCreateTableConflictHandling} - - + `exist_ok` / `existOk` does not append the provided data to an existing table. Use [`table.add()`](/ingestion/insert) for that. If you need to ensure a table exists *and* -contains specific rows, prefer the [empty-table-then-add pattern](#create-empty-table). +contains specific rows, choose an explicit [merge policy](/ingestion/merge-insert). Creating an empty table +and repeatedly appending the same batch does not deduplicate it. ### From a custom schema {#from-a-custom-schema} @@ -138,7 +143,7 @@ You can define a custom Arrow schema for the table. This is useful when you want -An explicit schema is also where you control nullability. If later writes omit a +An explicit schema also determines which columns accept missing values (`null`). If later writes omit a non-nullable column, or provide actual nulls for it, ingestion fails; nullable columns can be omitted or written with null values. Without an explicit schema, Python infers list-like vector values as fixed-size `float32` vector fields from the observed dimension. @@ -186,10 +191,8 @@ The **`vector`** column needs to be a [Vector](/integrations/data/pydantic#vecto ### From a Polars DataFrame {#from-a-polars-dataframe} Python Only -LanceDB supports [Polars](https://pola.rs/), a modern, fast DataFrame library -written in Rust. Just like in Pandas, the Polars integration is enabled by PyArrow -under the hood. A deeper integration between LanceDB Tables and Polars DataFrames -is on the way. +Pass a Polars DataFrame as input. Install `polars` for this example; the +conversion preserves the table structure through Arrow. @@ -263,12 +266,14 @@ for a `created_at` field. -When you run this code it, should raise the `ValidationError`. +The example catches a `ValidationError` for the timestamp with a different +timezone. A validator runs when the Pydantic model is constructed; declaring a +model as the table schema does not make every later write run that validator. -Datasets too large to pass to `create_table` at once are covered separately -- -see [Insert data](/ingestion/insert#large-datasets), which explains why -`table.add()` parallelizes where `create_table` does not. +Datasets too large to pass to `create_table` at once are covered separately; +see [Insert data](/ingestion/insert#large-datasets) for batches, file-backed +input and write parallelism. ## Create empty table {#create-empty-table} @@ -292,9 +297,8 @@ An empty table can be initialized via an Arrow schema. -Alternatively, you can also use Pydantic to specify the schema for the empty table. Note that we do not -directly import `pydantic` but instead use `lancedb.pydantic` which is a subclass of `pydantic.BaseModel` -that has been extended to support LanceDB specific types like `Vector`. +Alternatively, define the schema with `LanceModel`, a Pydantic model class +provided by `lancedb.pydantic`. Its `Vector` type declares a fixed vector dimension. @@ -342,6 +346,46 @@ Use the `drop_table()` method on the database to remove a table. -This permanently removes the table and is not recoverable, unlike deleting rows. -By default, if the table does not exist an exception is raised. To suppress this, -you can pass in `ignore_missing=True`. +Dropping a table removes the table object. Deleting rows keeps the table and +changes its contents; retained versions and cleanup are described in +[Cleanup](/maintenance/cleanup). +On a local database, dropping a table that does not exist raises an error; in +Python, pass `ignore_missing=True` to suppress it. A deployment may report +success instead, depending on how it stores its catalog, so do not rely on the +error to tell you a table was missing. Python's deployment connection does not +take `ignore_missing`. + + + +## Complete write workflow {#complete-example} + + + +Each program creates a temporary local database, performs the five operations, +prints the result, and removes its temporary data when finished. + +- **Python:** install `lancedb`, save the code as `ingestion.py`, and run + `python ingestion.py`. +- **TypeScript:** install `@lancedb/lancedb`, save the code as `ingestion.mts`, + and run `node ingestion.mts` with Node.js 22.18 or later. +- **Rust:** download the [complete Rust project](/static/code/ingestion-rust.zip), + extract it, and run `cargo run --locked`. It includes `src/main.rs`, the Cargo + manifest and a lockfile for the example's dependencies. The code is shown below. + + + +{PyWriteWorkflow} + + +{TsWriteWorkflowBody} +{TsWriteWorkflowTail} + + +{RsWriteWorkflow} + + + +After the operations, Alice has 10 logins, Bobby has 21, and Charlie has 6. +Dana is selected for deletion and is absent from the final result. + + diff --git a/docs/web/ingestion/delete.mdx b/docs/web/ingestion/delete.mdx index 0e2a4fdfa..b7b1f9432 100644 --- a/docs/web/ingestion/delete.mdx +++ b/docs/web/ingestion/delete.mdx @@ -1,77 +1,110 @@ --- title: "Delete rows" sidebarTitle: "Delete" -description: "Remove rows by predicate, and reclaim their space." +description: "Select and delete records, verify the remaining rows, and understand storage retention." icon: trash +availability: + oss: available + enterprise: available + summary: >- + Delete rows through the SDKs in OSS and Enterprise. Storage reclamation follows compaction and retention policy. --- -These examples use the table set up in -[Ingesting data](/ingestion), and the connection made there. +import { RsExample as RsDeleteRows } from '/snippets/ingestion_delete.mdx'; +import { TsDeleteRowsBody, TsDeleteRowsTail } from '/snippets/ingestion_examples.mdx'; +import { PyDeleteRows } from '/snippets/ingestion_examples.mdx'; +import { PyWriteWorkflow, TsWriteWorkflowBody, TsWriteWorkflowTail, RsWriteWorkflow } from '/snippets/ingestion_examples.mdx'; -import { - PyDeleteOperation as DeleteOperation, - PyUpdateOptimizeCleanup as UpdateOptimizeCleanup, - TsDeleteOperation, - TsUpdateOptimizeCleanup, - RsDeleteOperation, - RsUpdateOptimizeCleanup, -} from '/snippets/tables.mdx'; +Deleting rows removes matching records from the current table contents. It +keeps the table and schema so you can continue reading and writing. To remove +the table itself, see [Drop a table](/ingestion/create-table#drop-a-table). - -In SQL this is [`DELETE`](/sql/dml/delete). - +## Inspect the rows you intend to delete {#select-rows} -## Delete rows {#delete-rows} +Use the same predicate for inspection and deletion. The example below creates +Alice, Bob and Charlie, prints Charlie's row, and deletes it by `id = 3`. +Inspection and deletion are separate operations; concurrent writes can change +which rows match between them. -Delete operations **soft delete** rows that match a given condition. -The underlying data is not immediately removed, but is marked -for deletion (in the [deletion files](https://lance.org/format/table/#deletion-files) at the Lance format level) and excluded from query results. +## Delete matching records {#delete-rows} + +Each program creates Alice, Bob and Charlie in a temporary local database, +deletes Charlie, and prints the remaining rows: **Alice and Bob**. The temporary +database is removed when the program finishes. + +- **Python:** install `lancedb`, save the code as `delete_rows.py`, and run + `python delete_rows.py`. +- **TypeScript:** install `@lancedb/lancedb`, save the code as `delete_rows.mts`, + and run `node delete_rows.mts` with Node.js 22.18 or later. +- **Rust:** download the [Rust examples](/static/code/ingestion-feedback-rust.zip), + extract the project, and run `cargo run --locked --bin delete`. - - {DeleteOperation} - - - - {TsDeleteOperation} - - - - {RsDeleteOperation} - + +{PyDeleteRows} + + +{TsDeleteRowsBody} +{TsDeleteRowsTail} + + +{RsDeleteRows} + -Expected table contents: +Enterprise also supports [`DELETE`](/sql/dml/delete). -| id | name | login_count | -| --- | --- | --- | -| 1 | Alice | 10 | -| 2 | Bob | 20 | +## Verify that the rows are absent {#verify} +Query the same predicate after deletion and check the remaining row count. +A reader still using an older snapshot may see the earlier contents; see +[Consistency](/tables/consistency). - -**Deleting rows removes them from the index** +Deleted rows are excluded from current query results, including indexed +searches. Deletion does not imply immediate physical erasure of every stored +copy of the data. -When rows are deleted, those rows are also excluded from the index segments, so indexed queries will not return them either. If ALL the rows are deleted (i.e., the table is emptied), ensure that you recreate the index after ingesting new data. - +## Understand storage reclamation {#storage} -To permanently remove deleted rows, you can optimize the table, which will run compaction and cleans up the soft-deleted rows, which frees up storage space. +Compaction rewrites the live data layout; cleanup removes eligible retained +versions and files. Retained history can continue to reference data that no +longer appears in the latest table version. -- In LanceDB OSS, compaction and cleanup are manual. Run `table.optimize()` regularly to free up disk space. -- In LanceDB Enterprise, files aren't cleaned up by default. You can configure automatic compaction and cleanup behavior at cluster setup time to suit your organization's retention policy. +In OSS, `optimize()` provides maintenance operations. In Enterprise, managed +maintenance and retention follow the service configuration. Choose retention +according to the history you need, rather than shortening it as part of a +routine delete example. See [Compaction](/maintenance/compaction) and +[Cleanup](/maintenance/cleanup) for the separate maintenance workflow. -By default, table cleanup removes data up to 7 days ago. If you need to reclaim space from deleted rows more aggressively, manually call `table.optimize()` use a shorter retention window as follows: +## Complete write workflow {#complete-example} + + + +Each program creates a temporary local database, performs the five operations, +prints the result, and removes its temporary data when finished. + +- **Python:** install `lancedb`, save the code as `ingestion.py`, and run + `python ingestion.py`. +- **TypeScript:** install `@lancedb/lancedb`, save the code as `ingestion.mts`, + and run `node ingestion.mts` with Node.js 22.18 or later. +- **Rust:** download the [complete Rust project](/static/code/ingestion-rust.zip), + extract it, and run `cargo run --locked`. It includes `src/main.rs`, the Cargo + manifest and a lockfile for the example's dependencies. The code is shown below. - - {UpdateOptimizeCleanup} - - - - {TsUpdateOptimizeCleanup} - - - - {RsUpdateOptimizeCleanup} - + +{PyWriteWorkflow} + + +{TsWriteWorkflowBody} +{TsWriteWorkflowTail} + + +{RsWriteWorkflow} + + +After the operations, Alice has 10 logins, Bobby has 21, and Charlie has 6. +Dana is selected for deletion and is absent from the final result. + + diff --git a/docs/web/ingestion/index.mdx b/docs/web/ingestion/index.mdx index 2a8de6759..c37b46a62 100644 --- a/docs/web/ingestion/index.mdx +++ b/docs/web/ingestion/index.mdx @@ -1,125 +1,97 @@ --- title: "Ingesting data" sidebarTitle: "Overview" -description: "Getting data into a table: creating one from your data, and the four write operations that follow." +description: "Choose how to create a table, append records or change existing data." icon: inbox keywords: ["ingest", "create table", "add", "insert", "arrow", "pandas", "polars", "pydantic"] +availability: + oss: available + enterprise: available + summary: >- + Create tables and write records through the SDKs in OSS and Enterprise. Enterprise also provides SQL write statements. --- -import { - PyUpdateConnectEnterprise as UpdateConnectEnterprise, - PyUpdateConnectLocal as UpdateConnectLocal, - PyUpdateExampleTableSetup as UpdateExampleTableSetup, - TsUpdateConnectEnterprise, - TsUpdateConnectLocal, - TsUpdateExampleTableSetup, - RsUpdateConnectEnterprise, - RsUpdateConnectLocal, - RsUpdateExampleTableSetup, - RsUpdateMakeUsersReader, -} from '/snippets/tables.mdx'; +import { PyWriteWorkflow, TsWriteWorkflowBody, TsWriteWorkflowTail, RsWriteWorkflow } from '/snippets/ingestion_examples.mdx'; -In LanceDB, tables store records with a defined schema that specifies column -names and types. Across the SDKs, you can create tables from row-oriented data -and Apache Arrow data structures. The Python SDK additionally supports PyArrow -schemas for explicit schema control, and `LanceModel` for Pydantic-based -validation. +Ingestion brings data into a LanceDB table. Start by choosing a schema and +creating the table, then choose a write operation according to how incoming +records relate to data already stored. -Getting data in has two halves: creating a table from data you already have, and -writing to a table that exists. This page covers the first and introduces the -second. +## Choose a write operation {#choose-a-write-method} - -Every operation here has a SQL form -- see [DML](/sql#dml). - - -Start by [creating a table](/ingestion/create-table); the rest of this -section writes to one that exists. - -## Choose a write method {#choose-a-write-method} - -| Family | Method | Use this when... | -| --------------- | --------------- | ---------------- | -| [`add`](/ingestion/insert) | `add(data)` | You are appending new rows and do not need to match against existing ones. | -| `update` | `update(where=..., values=...)` | You want to edit rows that already exist, using a SQL filter. | -| `merge_insert` | `.when_matched_update_all()` | You have incoming rows and want to update keys that already exist in the table. | -| `merge_insert` | `.when_not_matched_insert_all()` | You have incoming rows and want to insert keys that do not exist yet. | -| `merge_insert` | `.when_matched_update_all()` + `.when_not_matched_insert_all()` | You want both behaviors together (often called **upsert**: update existing keys **and** insert missing keys in the same operation). | -| `merge_insert` | `.when_not_matched_by_source_delete(...)` | You want to remove target rows that are missing from the incoming source set. | - -Write operations return status metadata. For example, `update` reports updated row count and -committed `version`, `delete` reports deleted row count and `version`, and `merge_insert` reports -inserted, updated, deleted, retry-attempt, and `version` fields. Writes require a mutable table -handle; after checking out an older version for reads, call `checkout_latest` / `checkoutLatest` -before modifying data. -The committed `version` advances even for writes that affect zero rows, such as a delete predicate -that matches nothing. - -## Connect to LanceDB {#connect-to-lancedb} - -Connect to your local LanceDB instance: - - - - {UpdateConnectLocal} - - - - {TsUpdateConnectLocal} - - - - {RsUpdateConnectLocal} - - - -Or, connect to LanceDB Enterprise: - - - - {UpdateConnectEnterprise} - - - - {TsUpdateConnectEnterprise} - - - - {RsUpdateConnectEnterprise} - - - -In the Rust snippets, a `make_users_reader` helper is used to build Arrow input data. - - - - {RsUpdateMakeUsersReader} - - - -## Create the example table {#create-the-example-table} - -We'll start by creating a simple table with `id`, `name`, and `login_count` columns. All examples below use the same table. - - - {UpdateExampleTableSetup} - - - - {TsUpdateExampleTableSetup} - - - - {RsUpdateExampleTableSetup} - - - -Expected table contents: - -| id | name | login_count | +| Task | Operation | What happens | | --- | --- | --- | -| 1 | Alice | 10 | -| 2 | Bob | 20 | +| [Create a table](/ingestion/create-table) | `create_table` / `createTable` | Create a table from data or an explicit schema. | +| [Append records](/ingestion/insert) | `add` | Add rows without matching existing keys. | +| [Upsert by key](/ingestion/merge-insert) | `merge_insert` / `mergeInsert` | Choose whether matching rows are updated and new keys inserted. | +| [Update selected rows](/ingestion/update) | `update` | Change values in rows selected by a predicate. | +| [Delete selected rows](/ingestion/delete) | `delete` | Remove matching rows from the current table contents. | -The example above shows a PyArrow schema. You can just as well create the table using other -table creation patterns (Pandas, Polars, Pydantic, iterators, etc.) -- see [Create a table](/ingestion/create-table#create-a-table-with-data). +Appending the same input twice can create duplicate records. Use merge insert +with a stable key when the incoming data should update existing records. + +## Connect and prepare the data {#connect-to-lancedb} + +Use the [connection guide](/basics/connect) to connect to OSS or Enterprise. +The SDK write operations in this section work with both. Enterprise also +supports [SQL write statements](/sql#dml); an SDK filter expression does not +require a separate SQL connection. + +Choose the column types, required fields and vector dimensions before a large +load. SDKs support different input representations, including row objects and +Arrow batches. The [table creation guide](/ingestion/create-table) covers +those choices. + +## Try a complete write workflow {#create-the-example-table} + +This example starts with Alice and Bob, appends Charlie, updates Bob's name +and login count, then merges a batch containing Charlie and Dana. Finally, it +selects Dana's row for inspection and deletes it. + + + +Each program creates a temporary local database, performs the five operations, +prints the result, and removes its temporary data when finished. + +- **Python:** install `lancedb`, save the code as `ingestion.py`, and run + `python ingestion.py`. +- **TypeScript:** install `@lancedb/lancedb`, save the code as `ingestion.mts`, + and run `node ingestion.mts` with Node.js 22.18 or later. +- **Rust:** download the [complete Rust project](/static/code/ingestion-rust.zip), + extract it, and run `cargo run --locked`. It includes `src/main.rs`, the Cargo + manifest and a lockfile for the example's dependencies. The code is shown below. + + + +{PyWriteWorkflow} + + +{TsWriteWorkflowBody} +{TsWriteWorkflowTail} + + +{RsWriteWorkflow} + + + +After the operations, Alice has 10 logins, Bobby has 21, and Charlie has 6. +Dana is selected for deletion and is absent from the final result. + + + +## Verify the result {#verify} + +After a write completes, inspect the affected keys and fields and check the row +count. SDK return values differ by operation and language; use the relevant +API reference when relying on returned counts or version metadata. + +Readers with older snapshots may not see another handle's writes immediately. +See [Consistency](/tables/consistency). A table checked out at a historical +version must return to the latest version before it can be modified. + +## Load larger datasets {#large-loads} + +Use batches or file-backed input when the dataset is too large to hold in +memory. The [insert guide](/ingestion/insert#large-datasets) covers input +formats, progress reporting and write parallelism. Keep retry behavior in mind: +repeating an append is not the same as retrying a key-based upsert. diff --git a/docs/web/ingestion/insert.mdx b/docs/web/ingestion/insert.mdx index 2d411eb3b..f2ee0d7db 100644 --- a/docs/web/ingestion/insert.mdx +++ b/docs/web/ingestion/insert.mdx @@ -1,217 +1,237 @@ --- title: "Insert data" sidebarTitle: "Insert" -description: "Append rows to a table, including datasets too large to hold in memory." +description: "Append records, load file-backed data or batches, and verify ingestion." icon: plus +availability: + oss: available + enterprise: available + summary: >- + Append records through the SDKs in OSS and Enterprise. For large Enterprise writes, see [large upload settings](#bulk-ingests-into-a-remote-table). --- -Rows are appended with `table.add()`. It accepts the same shapes -[table creation](/ingestion) does — lists of objects, Arrow tables, pandas -and polars frames, Pydantic models — and commits a new table version. - -These examples use the table set up in [Ingesting data](/ingestion). - +import { + TsCreateTableFromIterator as TsCreateTableFromIterator, + RsCreateTableFromIterator as RsCreateTableFromIterator, +} from '/snippets/tables.mdx'; import { PyAddFromDataset as AddFromDataset, PyCreateTableFromIterator as CreateTableFromIterator, - TsCreateTableFromIterator as TsCreateTableFromIterator, - RsCreateTableFromIterator as RsCreateTableFromIterator, - TsAddProgress, -} from '/snippets/tables.mdx'; +} from '/snippets/ingestion_examples.mdx'; - -In SQL this is [`INSERT`](/sql/dml/insert), including `INSERT ... SELECT` to move rows server-side. - +import { RsExample as RsAddProgress } from '/snippets/ingestion_progress.mdx'; +import { PyAddProgress, TsAddProgressBody, TsAddProgressTail } from '/snippets/ingestion_examples.mdx'; +import { PyAppendRows } from '/snippets/ingestion_examples.mdx'; +import { PyWriteWorkflow, TsWriteWorkflowBody, TsWriteWorkflowTail, RsWriteWorkflow } from '/snippets/ingestion_examples.mdx'; -## Ingesting large datasets {#large-datasets} +Use `add()` to append records to an existing table. It does not compare incoming +keys with stored keys. Choose [merge insert](/ingestion/merge-insert) when the +same record may arrive again and should update an existing row. -When ingesting large datasets, use `table.add()` on an existing table rather than -passing all data to `create_table()`. The `add()` method auto-parallelizes large -writes, while `create_table(name, data)` does not. +## Append a first batch {#append} - -For best performance with large datasets, create an empty table first and then call -`table.add()`. This enables automatic write parallelism for materialized data sources. - +Install the Python SDK with `pip install lancedb`. This complete example starts +with Alice, appends Bob and Charlie, and prints all three records. -### From files (Parquet, CSV, etc.) {#from-files-parquet-csv-etc} -Python Only + +{PyAppendRows} + -For file-based data, pass a `pyarrow.dataset.Dataset` to `table.add()`. This streams -data from disk without loading the entire dataset into memory. +The program uses temporary local storage. For a complete TypeScript or Rust +example, see [Complete write workflow](#complete-example). +Enterprise SQL offers [`INSERT`](/sql/dml/insert), including `INSERT ... SELECT`. + +## Prepare the input {#prepare} + +Match the table's schema, including required fields and vector dimensions. +Supported input formats differ by SDK; see +[Create a table](/ingestion/create-table). Inspect conversion errors before +retrying, and verify the final row count and representative values. + +## Choose input for a large load {#large-datasets} + +Avoid building one large in-memory list when records already live in files +or arrive as batches. Create a table with the intended schema, then append +through the appropriate input path. + + +### Read from files {#from-files-parquet-csv-etc} + +Python accepts a `pyarrow.dataset.Dataset`. This example writes a small Parquet +file locally, creates an empty table with its schema, and appends the two records. +For your own data, point the dataset reader at the input files instead. + + +{AddFromDataset} + + +A file-backed source can be scanned without first loading the whole dataset +into application memory. This Python input API is distinct from converting +files to Arrow batches yourself in another language. + + +### Produce batches during ingestion {#from-iterators-custom-batch-generation} + +Use batches when processing the input as it arrives. The Python example creates +an empty table and appends ten records produced by a batch iterator. - - {AddFromDataset} - + +{CreateTableFromIterator} + + +{TsCreateTableFromIterator} + + +{RsCreateTableFromIterator} + - -`pa.dataset()` input is currently Python-only. TypeScript and Rust support for -file-based dataset ingestion is tracked in -[lancedb#3173](https://github.com/lancedb/lancedb/issues/3173). - +The language APIs differ: the TypeScript example appends each constructed Arrow +batch, while the Rust example supplies an Arrow reader when creating the table. +These are input-format examples, not equivalent parallel-write benchmarks. -### From iterators (custom batch generation) {#from-iterators-custom-batch-generation} +### Control write parallelism {#write-parallelism} -When you need custom batch logic — generating embeddings on the fly, transforming -rows from an external source, etc. — use an iterator of `RecordBatch` objects. +Python's `add()` can estimate parallelism for materialized inputs. A streaming +reader may not expose its total size or support replay. Use `write_parallelism` +when controlling the number of concurrent partitions; increasing it can also +increase memory use. TypeScript does not expose that manual option. + +Measure with realistic row sizes and storage. Avoid one write per row when a +batch will do, and avoid a large worker count for a small input. + +## Track completion and progress {#tracking-ingestion-progress} + +Python, TypeScript and Rust support progress callbacks when adding data. +These examples start with Alice, append Bob and Charlie, and print the number +of rows written as progress updates arrive. After the write succeeds, they +query the table to show all three records. + +Each program uses a temporary local database and removes it when finished. + +- **Python:** install `lancedb`, save the code as `insert_progress.py`, and run + `python insert_progress.py`. +- **TypeScript:** install `@lancedb/lancedb`, save the code as `insert_progress.mts`, + and run `node insert_progress.mts` with Node.js 22.18 or later. +- **Rust:** download the [Rust examples](/static/code/ingestion-feedback-rust.zip), + extract the project, and run `cargo run --locked --bin progress`. - - {CreateTableFromIterator} - - - - {TsCreateTableFromIterator} - - - - {RsCreateTableFromIterator} - + +{PyAddProgress} + + +{TsAddProgressBody} +{TsAddProgressTail} + + +{RsAddProgress} + -Use this pattern when: +For a progress bar in Python, install `tqdm` and pass `progress=True` instead +of a callback. -- Your source data already arrives in Arrow batches, readers, datasets, or streams. -- Materializing the entire ingest as one giant in-memory list or array would be too expensive. -- You want to control chunk size explicitly during ingestion. +### Read progress updates -Python can also consume iterators of other supported types like Pandas DataFrames or Python lists. +Updates include row and byte counts, elapsed time, and parallel write activity. +The field names differ by language: -### Write parallelism {#write-parallelism} +| Meaning | Python dictionary key | TypeScript property | Rust method | +|:--------|:----------------------|:--------------------|:------------| +| Rows written so far | `output_rows` | `outputRows` | `output_rows()` | +| Bytes written so far | `output_bytes` | `outputBytes` | `output_bytes()` | +| Expected row count, when known | `total_rows` | `totalRows` | `total_rows()` | +| Elapsed time | `elapsed_seconds` | `elapsedSeconds` | `elapsed()` | +| Active write tasks | `active_tasks` | `activeTasks` | `active_tasks()` | +| Total write tasks | `total_tasks` | `totalTasks` | `total_tasks()` | +| Final update | `done` | `done` | `done()` | - -For materialized data (`pa.Table`, `pd.DataFrame`, `pa.dataset()`), LanceDB -automatically parallelizes large writes — no configuration needed. Auto-parallelism -targets approximately 1M rows or 2GB per write partition. +Python and TypeScript report elapsed time in seconds; Rust returns a `Duration`. +The expected row count may be unknown for streaming input. On the final update, +it falls back to the number of rows written. -For streaming sources (iterators, `RecordBatchReader`), LanceDB cannot determine -total size upfront. Python exposes `write_parallelism` on both synchronous and -asynchronous `table.add()` calls; see the [iterator example](#iterator-ingestion-for-data-you-transform-on-the-fly). -TypeScript does not expose a manual write-parallelism option. - +Keep callbacks short and use the write operation's result to determine success. +The final `done` update means progress reporting has ended; it can also occur +after a write fails or is cancelled. Input errors before writing starts may +produce no updates. -### Tracking ingestion progress {#tracking-ingestion-progress} -TypeScript example +TypeScript delivers callbacks asynchronously. Slow callbacks can queue up, +and the `add()` promise may resolve before every callback has run. Exceptions +from a TypeScript callback are logged and do not abort the write. -In TypeScript, for long-running writes, pass a `progress` callback to `table.add()` to surface -per-batch progress in your UI, logs, or metrics pipeline. Progress updates are dispatched -asynchronously to the JavaScript event loop, including a final `done: true` -update when an active write finishes. +## Verify writes and handle retries {#performance} -Each invocation receives a `WriteProgress` object: +Wait for the write operation's result, then check the expected records. +Progress callbacks describe activity; they do not establish that a write +succeeded. If a request fails after the server may have committed it, blindly +repeating an append can duplicate rows. The client's automatic retries can do +the same: on a remote table, the Python and TypeScript 0.40.0 clients resend an +append after a retryable status such as `504`, even when the server had already +committed it, and `add()` still returns success. Reconcile using your record +IDs, or choose an upsert policy designed for replay. -| Field | Description | -|:------|:------------| -| `outputRows` | Rows written so far. | -| `outputBytes` | Bytes written so far. | -| `totalRows` | Expected total rows when the input source reports one. Always set on the final callback. | -| `elapsedSeconds` | Wall-clock seconds since the write started. | -| `activeTasks` | Parallel write tasks currently in flight. | -| `totalTasks` | Total parallel write tasks (the write parallelism). | -| `done` | `true` only for the final callback. | - - - - {TsAddProgress} - - - -A few things to know before you wire this up: - -- Asynchronous delivery: slow callbacks queue updates without back-pressuring the writer. Keep the callback cheap to avoid accumulating queued updates. -- Completion: an active write emits a final `done: true` update when it finishes, including execution errors or cancellation. Errors before writing starts, such as invalid input, may produce no progress callbacks. Use the `table.add()` promise to determine whether the write succeeded; it does not wait for JavaScript to process every queued callback. -- Errors swallowed: anything your callback throws is logged with `console.warn` and won't abort the write, so keep the callback side-effect-only and don't rely on it for control flow. -- Row totals: `totalRows` is only populated when the input source can report it up front (for example, a materialized `arrow.Table`). For streaming sources it stays `undefined` until the final callback, where it falls back to the actual rows written. - -## Performance {#performance} - -If ingestion is taking longer than expected on a large dataset, the cause is almost always how `add()` is called: each call commits a new version and a new fragment, so a per-row loop pays that per-call overhead at every row. The best practice is to pick the ingestion mode that matches your data shape — bulk ingestion when the data is already materialized, or iterator ingestion when it's streamed or computed on the fly. - - -**Why `merge_insert()` is significantly slower than `add()`** - -A merge has to scan existing data to find matches on the join key (or look them up via a scalar index, if one exists), and then delete-and-reinsert any updated rows in a single transaction; `add()` simply appends new fragments. - -Use `add()` for pure appends, and reach for `merge_insert()` only when you need upsert or conditional-insert logic. When you do use it, build a scalar index on the join column first — otherwise the match step falls back to a full column scan, which is the dominant cost at scale. - - -#### Bulk ingestion: for data you already have {#bulk-ingestion-for-data-you-already-have} - -For materialized inputs (Arrow Tables, DataFrames) and file-backed sources (`pyarrow.dataset(...)`), LanceDB auto-parallelizes the write across workers, estimating the partition count from the data size — more partitions means more concurrent writes and higher throughput, up to the CPU core count. - -```python Python icon="python" -table.add(arrow_table) # in-memory -table.add(df) # pandas -table.add(ds.dataset("data/", format="parquet")) # streams from disk, still parallelized -``` - -Pass `progress=True` to watch it happen: LanceDB shows a live tqdm bar with rows written, throughput in MB/s, and active worker count. - -```python Python icon="python" -table.add(ds.dataset("data/", format="parquet"), progress=True) -``` - -```text -71%|███████▏ | 710000/1000000 [00:12<00:05, 58.8kit/s, 42.3 MB/s | 8/8 workers] -``` - -For larger-than-memory data, prefer scanning a file-backed dataset (`ds.dataset(...)`) over a hand-built `pyarrow.RecordBatchReader`: a `Dataset` can be counted and rescanned, so LanceDB knows the row count upfront (better auto-parallelism) and can retry a failed write from the start — a reader can only be consumed once, so neither is possible. Reach for the iterator path below only when the data genuinely can't be backed by files, e.g. to apply custom data transformations as you ingest. - -For very large initial loads, create the table empty first; passing data directly to `create_table(name, data)` skips the auto-parallel path. - -#### Iterator ingestion: for data you transform on the fly {#iterator-ingestion-for-data-you-transform-on-the-fly} - -If each row needs work before it can be written — applying a custom transformation as you ingest, for example — the data doesn't exist as a file you can point `ds.dataset(...)` at. The best practice is to pass an iterator of `pyarrow.RecordBatch` instead; LanceDB consumes one batch at a time as you produce them. - -```python Python icon="python" -def stream(): - for raw in source: - vectors = model.encode(raw["text"]) - yield pa.RecordBatch.from_pydict({**raw, "vector": vectors}) - -table.add(stream()) -``` - -Use decently large chunks of several thousand rows or more, rather than yielding single-row batches. - - -**Set `write_parallelism` yourself for large inputs** - -A reader can't be counted or rescanned the way a `Dataset` can (see above), so LanceDB can't auto-size parallelism for it — set `write_parallelism` explicitly for large inputs: - -```python Python icon="python" -table.add(stream(), write_parallelism=4) -``` - -Each partition becomes its own fragment, so don't over-allocate on a small input — budget one unit of parallelism per ~100K rows or ~1 GB of data as a rule of thumb. - - -#### Bulk ingests into a remote table {#bulk-ingests-into-a-remote-table} +## Configure large uploads {#bulk-ingests-into-a-remote-table} Enterprise This section applies only to remote tables (`db://` connections). Embedded (local) writes are unaffected. -When you write to a remote LanceDB Enterprise table, the client splits each write partition into one or more HTTP `insert` parts and uploads them under a single upload id. Each part is streamed rather than buffered, so peak memory stays bounded. But each part still has to finish within the client read timeout while the server writes it to object storage, so on very large ingests an oversized part can run past that timeout and surface as: +When you write to a remote LanceDB Enterprise table, the client sends the write as one HTTP `insert` request unless it splits the data into more than one partition, which it does for a large input it can measure or when you set [`write_parallelism`](#write-parallelism). A partitioned write goes up as a multipart write: the client cuts each partition into one or more `insert` parts and uploads them under a single upload id. Each part is streamed. Its size and duration still affect resource use and +timeouts: it has to finish within the client read timeout while the server writes it to object storage, so on very large ingests an oversized part can run past that timeout and surface as: ```text lancedb.remote.errors.HttpError: operation timed out ``` -Two environment variables control how parts are cut. Both are picked up automatically by the Python and TypeScript clients, and are the only way to tune this from those SDKs: +Two environment variables control how a multipart write's parts are cut. Both are picked up automatically by the Python and TypeScript clients, for clients that support these settings: | Variable | Default | What it controls | |----------|---------|------------------| | `LANCE_CLIENT_MAX_BYTES_PER_REQUEST` | 8 GiB | Maximum size of a single insert part, in LZ4-compressed Arrow IPC bytes. Set to `0` to disable splitting and send one request per partition. | | `LANCE_CLIENT_MAX_REQUEST_DURATION` | Half the read timeout | Maximum wall-clock time, in whole seconds, that any one insert part may stay open. Set to `0` to disable the time-based cut. | -A part is cut when it hits either limit, whichever comes first. Lower the byte budget when large ingests hit the read timeout; lower the duration when uploads are slow or throttled and the byte budget isn't the limiting factor. +A part is cut after the input batch that reaches either limit, so cuts fall between batches. Lower the byte budget, with smaller input batches, when large ingests hit the read timeout; lower the duration when uploads are slow or throttled and the byte budget isn't the limiting factor. ```bash export LANCE_CLIENT_MAX_BYTES_PER_REQUEST=1073741824 # 1 GiB export LANCE_CLIENT_MAX_REQUEST_DURATION=120 # 2 minutes ``` -Splitting into more parts does not change the final table. The server stages every part under the shared upload id and merges them atomically when the write completes. +Splitting into more parts does not change the final table. The server stages the parts under their shared upload ID before committing +the write. This upload protocol is separate from repeating an application-level +`add()` after a timeout. + +## Complete write workflow {#complete-example} + + + +Each program creates a temporary local database, performs the five operations, +prints the result, and removes its temporary data when finished. + +- **Python:** install `lancedb`, save the code as `ingestion.py`, and run + `python ingestion.py`. +- **TypeScript:** install `@lancedb/lancedb`, save the code as `ingestion.mts`, + and run `node ingestion.mts` with Node.js 22.18 or later. +- **Rust:** download the [complete Rust project](/static/code/ingestion-rust.zip), + extract it, and run `cargo run --locked`. It includes `src/main.rs`, the Cargo + manifest and a lockfile for the example's dependencies. The code is shown below. + + + +{PyWriteWorkflow} + + +{TsWriteWorkflowBody} +{TsWriteWorkflowTail} + + +{RsWriteWorkflow} + + + +After the operations, Alice has 10 logins, Bobby has 21, and Charlie has 6. +Dana is selected for deletion and is absent from the final result. + + diff --git a/docs/web/ingestion/merge-insert.mdx b/docs/web/ingestion/merge-insert.mdx index 8bfb6084a..6fa59fd9c 100644 --- a/docs/web/ingestion/merge-insert.mdx +++ b/docs/web/ingestion/merge-insert.mdx @@ -1,19 +1,16 @@ --- title: "Merge insert" sidebarTitle: "Merge Insert" -description: "Reconcile incoming rows against a key: update what matches, insert what does not." +description: "Match incoming records by key and choose which rows to update, insert or remove." icon: code-merge +availability: + oss: available + enterprise: available + summary: >- + Merge insert is available through the SDKs in OSS and Enterprise. Use it for key-based updates and inserts. --- -These examples use the table set up in -[Ingesting data](/ingestion), and the connection made there. - import { - PyMergeMatchedUpdateOnly as MergeMatchedUpdateOnly, - PyInsertIfNotExists as InsertIfNotExists, - PyMergeUpdateInsert as MergeUpdateInsert, - PyMergeDeleteMissingBySource as MergeDeleteMissingBySource, - PyMergePartialColumns as MergePartialColumns, TsMergeMatchedUpdateOnly, TsInsertIfNotExists, TsMergeUpdateInsert, @@ -25,15 +22,24 @@ import { RsMergeDeleteMissingBySource, RsMergePartialColumns, } from '/snippets/tables.mdx'; +import { + PyMergeMatchedUpdateOnly as MergeMatchedUpdateOnly, + PyInsertIfNotExists as InsertIfNotExists, + PyMergeUpdateInsert as MergeUpdateInsert, + PyMergeDeleteMissingBySource as MergeDeleteMissingBySource, + PyMergePartialColumns as MergePartialColumns, +} from '/snippets/ingestion_examples.mdx'; - -Merge insert has **no SQL form** -- it needs a source relation and a per-match policy that one statement cannot express. See [DML](/sql/dml/insert#next). - +import { PyWriteWorkflow, TsWriteWorkflowBody, TsWriteWorkflowTail, RsWriteWorkflow } from '/snippets/ingestion_examples.mdx'; + +Use merge insert when incoming records should be compared with existing rows. +Combine update and insert policies for an **upsert**. Use [append](/ingestion/insert) +when every incoming row should simply be added. ## Merge incoming rows by key {#merge-incoming-rows-by-key} Merging is different from updating because it involves comparing incoming rows to existing rows by key, and then choosing what to do based on whether the key exists in the target table or not. -The `merge_insert(""..."")` method lets you do this. +Use `merge_insert("id")` in Python or the corresponding SDK builder. In merge operations, rows are split into three groups: @@ -45,37 +51,28 @@ Conditional merge clauses can compare old and new values. Use the `target.` pref existing table row and `source.` for the incoming row, for example `target.last_update < source.last_update`. - -**Use scalar indexes to speed up merge insert** +Choose a key that identifies the same record in the source and target. Check +for duplicate and null keys before merging. A primary-key declaration does not +enforce uniqueness on ordinary writes, and append operations do not deduplicate +records for you. -The merge insert command performs a join between the input data and the target table `on` the key you provide. This requires scanning that entire column, which can be expensive for large tables. To speed up this operation, create a scalar index on the join column, which will allow LanceDB to find matches without scanning the whole table. +A scalar index on the key can help matching. Keep it current and measure the +actual workload rather than assuming every merge uses an index. Enterprise may +reject a merge when the unindexed scan exceeds its configured limit; inspect +that error and the index state before retrying. -Read more about scalar indices in the [Scalar Index](/indexing/scalar-index/) guide. - +## Choose a merge policy {#policies} - -If you see this HTTP 400 error from `merge_insert`: `Bad request: Merge insert cannot be performed because the number of unindexed rows exceeds the maximum of 10000`. Verify that the scalar index on the join column is up to date before retrying. - +| Source and target relationship | Available action | +| --- | --- | +| Key appears in both | Update matching target rows, optionally with a condition. | +| Key appears only in the source | Insert it, or leave it out. | +| Key appears only in the target | Keep it, or explicitly delete it under a synchronization policy. | - -**Rust: build merge predicates with DataFusion expressions** - -In the Rust SDK, you can pass a `datafusion_expr::Expr` directly instead of a SQL string by using -`when_matched_update_all_expr` and `when_not_matched_by_source_delete_expr` on `MergeInsertBuilder`. -This is useful when you are constructing predicates programmatically and want to avoid building a SQL string. - -These methods are only supported on local tables. Calling them against a remote table returns a -`NotSupported` error — use the SQL string variants (`when_matched_update_all` / -`when_not_matched_by_source_delete`) for remote tables. - - -Like the create table and add APIs, the merge insert API will automatically compute embeddings based on the [embedding registry](/embedding/index#embedding-registry) if the table has an embedding definition in its schema. - -During `merge_insert`, if the input data doesn't contain the source column (i.e., the original field used to generate embeddings, such as text for a text embedding model or `image_uri` for an image model), or if a vector value is already provided, LanceDB skips embedding generation for that row. Embeddings are only auto-generated when that source field is present in the incoming data, **and** the vector field is empty. - -Primary keys in LanceDB are metadata used by operations such as `merge_insert`; they are not -enforced as uniqueness constraints on ordinary writes. Keep using `merge_insert` or your own -deduplication logic when you need key-based upsert semantics. +The following examples are independent. Each starts from the table shown in +its code, so the expected results do not depend on running the previous example. +Python blocks include their own temporary connection. For a full TypeScript or +Rust program, see [Complete write workflow](#complete-example). ### Update matched rows only {#update-matched-rows-only} @@ -160,7 +157,10 @@ Expected table contents: ### Delete target rows that are missing from source {#delete-target-rows-that-are-missing-from-source} -Use `when_not_matched_by_source_delete()` when you want to remove any target row that does not appear in the incoming source data. +Use `when_not_matched_by_source_delete()` only when the incoming data is the +complete set for the target scope you intend to synchronize. An incremental +batch is usually not a complete source: deleting everything absent from it +could remove unrelated target records. Inspect the unmatched target keys first. @@ -183,7 +183,8 @@ Expected table contents: | 2 | Bobby | 21 | | 3 | Charlie | 5 | -In the example above, LanceDB matches rows by `id`. Rows with `id=2` and `id=3` exist in both the table and incoming data, so they are updated. Row `id=1` exists only in the target, so it is deleted. +Here, IDs 2 and 3 are updated and ID 1 is deleted. This policy synchronizes +the target to the supplied source set. ### Use partial columns in merge updates {#use-partial-columns-in-merge-updates} @@ -212,3 +213,54 @@ Expected table contents: | 3 | Charlie | null | Note that in the example above, when `merge_insert` creates a new row, any missing columns are written as `null`. If a missing column is non-nullable in your schema, the insert will fail. + +## Check the merged records {#verify} + +Inspect the keys that should have changed, confirm that unrelated rows remain, +and check inserted or deleted counts when the SDK returns them. Repeating a +merge is only safe when its keys, conditions and update values produce the +intended result on retry. + +### Embeddings and expression predicates {#advanced} + +When a table uses an [embedding definition](/embedding), behavior depends on +which source and vector fields the incoming records provide. Do not assume a +partial metadata update recomputes embeddings. Inspect the resulting vector +when changing the input used to generate it. + +Rust's `when_matched_update_all_expr` and +`when_not_matched_by_source_delete_expr` accept DataFusion expressions for +local tables. Use the string predicate variants for Enterprise tables. + +## Complete write workflow {#complete-example} + + + +Each program creates a temporary local database, performs the five operations, +prints the result, and removes its temporary data when finished. + +- **Python:** install `lancedb`, save the code as `ingestion.py`, and run + `python ingestion.py`. +- **TypeScript:** install `@lancedb/lancedb`, save the code as `ingestion.mts`, + and run `node ingestion.mts` with Node.js 22.18 or later. +- **Rust:** download the [complete Rust project](/static/code/ingestion-rust.zip), + extract it, and run `cargo run --locked`. It includes `src/main.rs`, the Cargo + manifest and a lockfile for the example's dependencies. The code is shown below. + + + +{PyWriteWorkflow} + + +{TsWriteWorkflowBody} +{TsWriteWorkflowTail} + + +{RsWriteWorkflow} + + + +After the operations, Alice has 10 logins, Bobby has 21, and Charlie has 6. +Dana is selected for deletion and is absent from the final result. + + diff --git a/docs/web/ingestion/update.mdx b/docs/web/ingestion/update.mdx index e77587ea8..d44a2561e 100644 --- a/docs/web/ingestion/update.mdx +++ b/docs/web/ingestion/update.mdx @@ -1,82 +1,123 @@ --- title: "Update rows" sidebarTitle: "Update" -description: "Change values in rows that already exist, by predicate or SQL expression." +description: "Select existing rows, change values or expressions, and verify the changes." icon: pen +availability: + oss: available + enterprise: available + summary: >- + Update rows through the SDKs in OSS and Enterprise. Enterprise also supports the SQL UPDATE statement. --- -These examples use the table set up in -[Ingesting data](/ingestion), and the connection made there. - import { - PyUpdateOperation as UpdateOperation, - PyUpdateUsingSql as UpdateUsingSql, TsUpdateOperation, TsUpdateUsingSql, RsUpdateOperation, RsUpdateUsingSql, } from '/snippets/tables.mdx'; +import { + PyUpdateOperation as UpdateOperation, + PyUpdateUsingSql as UpdateUsingSql, +} from '/snippets/ingestion_examples.mdx'; - -In SQL this is [`UPDATE`](/sql/dml/update). - +import { PyWriteWorkflow, TsWriteWorkflowBody, TsWriteWorkflowTail, RsWriteWorkflow } from '/snippets/ingestion_examples.mdx'; -## Update rows {#update-rows} +Use `update` to change rows selected by a predicate. Use +[merge insert](/ingestion/merge-insert) instead when matching an incoming dataset +against existing keys. -Use `update` when you already know which target rows to modify and you do not need to compare against an incoming dataset. +## Select the rows to change {#select-rows} + +Check the predicate with a [filtered query](/search/filtering) before applying +the update. These examples target the row with `id = 2`. Review the scope +carefully when omitting a predicate, since that can update the entire table. + +Each example below starts from Alice with 10 logins and Bob with 20. The +Python programs create and remove their own temporary database. TypeScript +and Rust operation blocks use `db`; a complete program is provided below. + +## Assign a new value {#update-rows} - - {UpdateOperation} - - - - {TsUpdateOperation} - - - - {RsUpdateOperation} - + +{UpdateOperation} + + +{TsUpdateOperation} + + +{RsUpdateOperation} + -Expected table contents: +The result contains Alice and **Bobby**, with their login counts unchanged. +The Python `values` and TypeScript `values` arguments take literal values. +Rust's `column` argument is an expression, so the string value includes SQL quotes. -| id | name | login_count | -| --- | --- | --- | -| 1 | Alice | 10 | -| 2 | Bobby | 20 | - - -Updating nested columns is not yet supported. - - -## Update rows with SQL expressions {#update-rows-with-sql-expressions} - -Use `values_sql` when you want to use SQL-like expressions to update rows. This is useful for operations like incrementing a counter, or setting a column value based on another column. +## Calculate a value from the existing row {#update-rows-with-sql-expressions} - - {UpdateUsingSql} - - - - {TsUpdateUsingSql} - - - - {RsUpdateUsingSql} - + +{UpdateUsingSql} + + +{TsUpdateUsingSql} + + +{RsUpdateUsingSql} + -Expected table contents: +In this separate example, Bob keeps his name and his login count becomes **21**. +Python's `values_sql` and TypeScript's `valuesSql` accept expressions. +These SDK expressions work with embedded OSS tables; they do not require an +Enterprise SQL endpoint. Enterprise also offers [`UPDATE`](/sql/dml/update). -| id | name | login_count | -| --- | --- | --- | -| 1 | Alice | 10 | -| 2 | Bob | 21 | +## Verify the affected data {#verify} - -See the [SQL queries](/sql#connecting) page for more information on the supported SQL syntax. - +Read the selected rows again and inspect the changed fields. For another +connection or table handle, account for its +[read consistency](/tables/consistency) settings. -When rows are updated, they are moved out of any existing index. The row will still show up in search queries, but the query will not be as fast as it would be if the row was in the index. If you update a large proportion of rows, consider triggering an index rebuild afterwards. +Nested-field updates are not supported by these operations. For schema changes, +see [schema evolution](/management/table#schema). Treat derived values separately: +changing an input field is not a guarantee that every stored feature has been +recomputed. See [Refresh a column](/feature-engineering/refresh-column). + +Updates can change index coverage and query cost. After a large update, +inspect the workload and follow [index maintenance](/indexing/reindexing) +guidance for the backend you use. + +## Complete write workflow {#complete-example} + + + +Each program creates a temporary local database, performs the five operations, +prints the result, and removes its temporary data when finished. + +- **Python:** install `lancedb`, save the code as `ingestion.py`, and run + `python ingestion.py`. +- **TypeScript:** install `@lancedb/lancedb`, save the code as `ingestion.mts`, + and run `node ingestion.mts` with Node.js 22.18 or later. +- **Rust:** download the [complete Rust project](/static/code/ingestion-rust.zip), + extract it, and run `cargo run --locked`. It includes `src/main.rs`, the Cargo + manifest and a lockfile for the example's dependencies. The code is shown below. + + + +{PyWriteWorkflow} + + +{TsWriteWorkflowBody} +{TsWriteWorkflowTail} + + +{RsWriteWorkflow} + + + +After the operations, Alice has 10 logins, Bobby has 21, and Charlie has 6. +Dana is selected for deletion and is absent from the final result. + + diff --git a/docs/web/integrations/ai/hermes-agent.mdx b/docs/web/integrations/ai/hermes-agent.mdx index a5521a9d8..621b53129 100644 --- a/docs/web/integrations/ai/hermes-agent.mdx +++ b/docs/web/integrations/ai/hermes-agent.mdx @@ -29,7 +29,7 @@ prompt, plus lexical (keyword) search over past sessions. Both are useful, but k misses paraphrases of what you originally typed — the exact thing you need when recalling a fact you phrased differently months ago. -LanceDB is an embedded retrieval library, which makes it a natural fit here: +LanceDB OSS is an embedded retrieval library, which makes it a natural fit here: - **No server to stand up** — it reads and writes a table on local disk, so the plugin ships as a dependency rather than a service to operate. diff --git a/docs/web/integrations/ai/huggingface.mdx b/docs/web/integrations/ai/huggingface.mdx index 62ac7e8cd..0a38c208b 100644 --- a/docs/web/integrations/ai/huggingface.mdx +++ b/docs/web/integrations/ai/huggingface.mdx @@ -2,6 +2,12 @@ title: "Hugging Face Hub" sidebarTitle: "Hugging Face" description: "Use LanceDB directly on Lance datasets hosted on the Hugging Face Hub for multimodal search and retrieval." +availability: + oss: available + enterprise: unavailable + summary: >- + The embedded library opens Lance datasets on the Hugging Face Hub in place + through `hf://` URIs. A deployment does not read `hf://` datasets. --- [Hugging Face Hub](https://huggingface.co/datasets?format=format:lance&sort=trending) is a popular platform for sharing machine learning datasets, models, and other resources. diff --git a/docs/web/integrations/ai/langchain.mdx b/docs/web/integrations/ai/langchain.mdx index 363331093..14ed628a8 100644 --- a/docs/web/integrations/ai/langchain.mdx +++ b/docs/web/integrations/ai/langchain.mdx @@ -80,6 +80,9 @@ It returns list of ids of the added texts. {PyFrameworksLangchainAddTexts} +With LanceDB Enterprise, `tbl.to_pandas()` raises in the synchronous Python client; +read the rows with a query instead, such as `tbl.search().limit(100).to_pandas()`. + ------ diff --git a/docs/web/integrations/data/duckdb.mdx b/docs/web/integrations/data/duckdb.mdx index 704a47c60..23c4891f9 100644 --- a/docs/web/integrations/data/duckdb.mdx +++ b/docs/web/integrations/data/duckdb.mdx @@ -6,7 +6,7 @@ description: "Learn how to use the DuckDB-Lance extension to query Lance tables LanceDB integrates with [DuckDB](https://duckdb.org/) through the [Lance extension](https://github.com/lance-format/lance-duckdb) for DuckDB. In this page, we'll show how LanceDB manages table lifecycle, and DuckDB provides SQL analytics (including joins) and search over those tables. -Note that earlier versions of LanceDB used to recommend converting Lance tables to Arrow tables via `table.to_arrow()`. Although this method is still available (because DuckDB [natively scans Arrow tables](https://duckdb.org/2021/12/03/duck-arrow)), it is no longer the recommended workflow for working with Lance tables in DuckDB. This page shows how to use the Lance extension with namespace-attached LanceDB tables, allowing you to pushdown SQL queries directly to the Lance layer. +Note that earlier versions of LanceDB used to recommend converting Lance tables to Arrow tables via `table.to_arrow()`. Although this method is still available for local tables (because DuckDB [natively scans Arrow tables](https://duckdb.org/2021/12/03/duck-arrow)), it is no longer the recommended workflow for working with Lance tables in DuckDB. This page shows how to use the Lance extension with namespace-attached LanceDB tables, allowing you to pushdown SQL queries directly to the Lance layer. ## Install {#install} diff --git a/docs/web/integrations/data/pandas_and_pyarrow.mdx b/docs/web/integrations/data/pandas_and_pyarrow.mdx index 170ccf772..c646e7f3b 100644 --- a/docs/web/integrations/data/pandas_and_pyarrow.mdx +++ b/docs/web/integrations/data/pandas_and_pyarrow.mdx @@ -1,6 +1,12 @@ --- title: "Pandas and PyArrow" sidebarTitle: "Pandas & PyArrow" +availability: + oss: available + enterprise: available + summary: >- + DataFrames go into `create_table` and come back from query results the same + way for local and deployment tables. --- diff --git a/docs/web/integrations/data/polars_arrow.mdx b/docs/web/integrations/data/polars_arrow.mdx index 4cb85737e..3acfcfea5 100644 --- a/docs/web/integrations/data/polars_arrow.mdx +++ b/docs/web/integrations/data/polars_arrow.mdx @@ -1,6 +1,13 @@ --- title: "Polars" sidebarTitle: "Polars & Arrow" +availability: + oss: available + enterprise: available + summary: >- + Polars data goes into `create_table`, and query results come back as Polars + DataFrames, for local and deployment tables alike; a LazyFrame over a whole + table is local only. --- @@ -36,7 +43,7 @@ Run vector search and keep the results as a Polars `DataFrame` for further proce ## Work with LazyFrames {#work-with-lazyframes} -When you want to operate on the entire table (potentially larger than RAM), convert to a Polars `LazyFrame` so you can chain transformations without loading everything at once. +When you want to operate on the entire table (potentially larger than RAM), convert to a Polars `LazyFrame` so you can chain transformations without loading everything at once. This works for local tables only; with LanceDB Enterprise, `search(...).to_polars()` returns an eager DataFrame of query results. {PyPlatformsPolarsLazyframe} diff --git a/docs/web/integrations/data/pydantic.mdx b/docs/web/integrations/data/pydantic.mdx index b6f80aca3..3ac660774 100644 --- a/docs/web/integrations/data/pydantic.mdx +++ b/docs/web/integrations/data/pydantic.mdx @@ -1,6 +1,13 @@ --- title: "Pydantic" sidebarTitle: "Pydantic" +availability: + oss: available + enterprise: available + summary: >- + The SDK converts `LanceModel` schemas to Arrow, and query rows back to + models, in your process, so they work the same with local and deployment + tables. --- diff --git a/docs/web/integrations/embedding/cohere.mdx b/docs/web/integrations/embedding/cohere.mdx index 8051462a9..b9a8e3378 100644 --- a/docs/web/integrations/embedding/cohere.mdx +++ b/docs/web/integrations/embedding/cohere.mdx @@ -8,6 +8,10 @@ import { PyEmbeddingCohereUsage } from '/snippets/integrations.mdx'; Using cohere API requires cohere package, which can be installed using `pip install cohere`. Cohere embeddings are used to generate embeddings for text data. The embeddings can be used for various tasks like semantic search, clustering, and classification. You also need to set the `COHERE_API_KEY` environment variable to use the Cohere API. +```shell +export COHERE_API_KEY="your-api-key" +``` + Supported models are: - embed-english-v3.0 diff --git a/docs/web/integrations/embedding/ibm.mdx b/docs/web/integrations/embedding/ibm.mdx index cec6c32bf..6dd3d57e3 100644 --- a/docs/web/integrations/embedding/ibm.mdx +++ b/docs/web/integrations/embedding/ibm.mdx @@ -3,7 +3,7 @@ title: IBM watsonx sidebarTitle: IBM WatsonX --- -import { PyEmbeddingIbmUsage } from '/snippets/integrations.mdx'; +import { PyEmbeddingIbmUsage } from '/snippets/provider_examples.mdx'; Generate text embeddings using IBM's watsonx.ai platform. @@ -54,9 +54,13 @@ Optionally set environment variables (if not passing credentials to `create` dir export WATSONX_API_KEY="YOUR_WATSONX_API_KEY" # Provide exactly one of the following: export WATSONX_PROJECT_ID="YOUR_WATSONX_PROJECT_ID" -export WATSONX_SPACE_ID="YOUR_WATSONX_SPACE_ID" +# export WATSONX_SPACE_ID="YOUR_WATSONX_SPACE_ID" ``` +Replace the credential placeholders with your provider credentials. The registry +variable keeps the API key out of stored embedding configuration. In application +code, load its value from environment variables or a secrets manager. + {PyEmbeddingIbmUsage} diff --git a/docs/web/integrations/embedding/jina.mdx b/docs/web/integrations/embedding/jina.mdx index 70457180d..588500d36 100644 --- a/docs/web/integrations/embedding/jina.mdx +++ b/docs/web/integrations/embedding/jina.mdx @@ -6,7 +6,7 @@ sidebarTitle: Jina import { PyEmbeddingJinaText, PyEmbeddingJinaMultimodal, -} from '/snippets/integrations.mdx'; +} from '/snippets/provider_examples.mdx'; ## Text Embedding Models {#text-embedding-models} @@ -23,6 +23,8 @@ Supported parameters (to be passed in `create` method) are: Usage Example: +Replace `your-api-key` with your Jina API key. This provider reads it from `JINA_API_KEY`; the example sets it explicitly. In application code, supply it through your environment or a secrets manager. + {PyEmbeddingJinaText} diff --git a/docs/web/integrations/embedding/openai.mdx b/docs/web/integrations/embedding/openai.mdx index cdd60d7cd..fbfa62d9d 100644 --- a/docs/web/integrations/embedding/openai.mdx +++ b/docs/web/integrations/embedding/openai.mdx @@ -10,9 +10,15 @@ LanceDB registers the OpenAI embeddings function in the registry by default, as | Parameter | Type | Default Value | Description | |---|---|---|---| | `name` | `str` | `"text-embedding-ada-002"` | The name of the model. | -| `dim` | `int` | Model default | For OpenAI's newer text-embedding-3 model, we can specify a dimensionality that is smaller than the 1536 size. This feature supports it | +| `dim` | `int` | Model default | Requested output dimensions for models that support shortening embeddings. Use `func.ndims()` when defining the vector field. | | `use_azure` | bool | `False` | Set true to use Azure OpenAI SDK | +Set your OpenAI API key before running the example: + +```shell +export OPENAI_API_KEY="your-api-key" +``` + {PyEmbeddingOpenaiBasic} diff --git a/docs/web/integrations/embedding/superlinked.mdx b/docs/web/integrations/embedding/superlinked.mdx index f35796230..bb2768d74 100644 --- a/docs/web/integrations/embedding/superlinked.mdx +++ b/docs/web/integrations/embedding/superlinked.mdx @@ -36,6 +36,9 @@ Supported parameters on `.create()`: ## Usage {#usage} ```py Python icon=Python +import tempfile +from pathlib import Path + import lancedb from lancedb.embeddings import get_registry from lancedb.pydantic import LanceModel, Vector @@ -47,22 +50,23 @@ sie = get_registry().get("sie").create( ) class Documents(LanceModel): + id: int text: str = sie.SourceField() vector: Vector(sie.ndims()) = sie.VectorField() -db = lancedb.connect("~/.lancedb") -table = db.create_table("docs", schema=Documents, mode="overwrite") +db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "superlinked-demo")) +table = db.create_table("docs", schema=Documents) table.add([ - {"text": "Machine learning is a subset of AI."}, - {"text": "Neural networks use multiple layers."}, - {"text": "Python is popular for ML development."}, + {"id": 1, "text": "Machine learning is a subset of AI."}, + {"id": 2, "text": "Neural networks use multiple layers."}, + {"id": 3, "text": "Python is popular for ML development."}, ]) results = table.search("What is deep learning?").limit(3).to_list() ``` -LanceDB handles embedding generation for both inserts and queries automatically, based on the `SourceField` / `VectorField` declarations on the schema. +LanceDB handles embedding generation for both inserts and queries automatically, based on the `SourceField` / `VectorField` declarations on the schema. The `id` column gives each document a stable key, which [entity extraction](#entity-extraction) uses to merge its results back. ## Hybrid search with reranker {#hybrid-search-with-reranker} @@ -88,7 +92,7 @@ for r in results: The reranker also works with pure vector or pure FTS search via `.rerank()`. ## ColBERT and multivector {#colbert-and-multivector} -`SIEMultiVectorEmbeddingFunction` (registered as `"sie-multivector"`) works with LanceDB's native `MultiVector` type and MaxSim scoring for ColBERT and ColPali models: +`SIEMultiVectorEmbeddingFunction` (registered as `"sie-multivector"`) works with LanceDB's native `MultiVector` type and MaxSim scoring for ColBERT and ColPali models. In LanceDB Python 0.40.0, a multivector that LanceDB generates on insert is stored as null unless the document's token count happens to equal the per-token dimension, so this example computes the document multivectors with `compute_source_embeddings()` and passes them in the `vector` column. LanceDB still generates the query multivector automatically: ```py Python icon=Python from lancedb.pydantic import MultiVector @@ -102,8 +106,12 @@ class ColBERTDocs(LanceModel): text: str = sie_colbert.SourceField() vector: MultiVector(sie_colbert.ndims()) = sie_colbert.VectorField() -table = db.create_table("colbert_docs", schema=ColBERTDocs, mode="overwrite") -table.add([{"text": "Machine learning is a subset of AI."}]) +table = db.create_table("colbert_docs", schema=ColBERTDocs) + +# Compute the document multivectors and store them with their text +texts = ["Machine learning is a subset of AI."] +vectors = sie_colbert.compute_source_embeddings(texts) +table.add([{"text": text, "vector": vector} for text, vector in zip(texts, vectors)]) # MaxSim search - query and document multivectors compared token-by-token results = table.search("What is ML?").limit(5).to_list() @@ -111,7 +119,7 @@ results = table.search("What is ML?").limit(5).to_list() ## Entity extraction {#entity-extraction} -`SIEExtractor` adds entity extraction to LanceDB's data-enrichment workflows. Extract entities from a text column and merge the results back as a structured Arrow column - enabling filtered search on extracted entities: +`SIEExtractor` adds entity extraction to LanceDB's data-enrichment workflows. Extract entities from a text column and merge the results back as a structured Arrow column - enabling filtered search on extracted entities. This example enriches the `docs` table from [Usage](#usage), matching rows on its `id` column: ```py Python icon=Python from sie_lancedb import SIEExtractor @@ -121,8 +129,9 @@ extractor = SIEExtractor( model="urchade/gliner_multi-v2.1", ) +docs = db.open_table("docs") extractor.enrich_table( - table, + docs, source_column="text", target_column="entities", labels=["person", "technology", "organization"], diff --git a/docs/web/integrations/embedding/voyageai.mdx b/docs/web/integrations/embedding/voyageai.mdx index 54d155d1b..8babfc214 100644 --- a/docs/web/integrations/embedding/voyageai.mdx +++ b/docs/web/integrations/embedding/voyageai.mdx @@ -11,6 +11,10 @@ Voyage AI provides cutting-edge embedding and rerankers. Using voyageai API requires voyageai package, which can be installed using `pip install voyageai`. Voyage AI embeddings are used to generate embeddings for text data. The embeddings can be used for various tasks like semantic search, clustering, and classification. You also need to set the `VOYAGE_API_KEY` environment variable to use the VoyageAI API. +```shell +export VOYAGE_API_KEY="your-api-key" +``` + Supported models are: - voyage-4-large (best retrieval quality, 1024 default dimensions, supports 256/512/1024/2048) diff --git a/docs/web/integrations/lerobotdataset.mdx b/docs/web/integrations/lerobotdataset.mdx index 08cf8bd15..cfb66b69b 100644 --- a/docs/web/integrations/lerobotdataset.mdx +++ b/docs/web/integrations/lerobotdataset.mdx @@ -143,9 +143,12 @@ class EmbedFromBlob: # SigLIP2 loads once per worker **Curation** is one query across all of it. Semantic search with a predicate on the derived score, no join, no second system: ```python -tbl.search(vec("a gripper closing on an object"), vector_column_name="emb_siglip2") \ - .where("jerk_score > 1.2875") # the roughest 1% of frames - .limit(4) +results = ( + tbl.search(vec("a gripper closing on an object"), vector_column_name="emb_siglip2") + .where("jerk_score > 1.2875") # the roughest 1% of frames + .limit(4) + .to_arrow() +) ``` **Mining** is the same query pointed at the rare cases. `tbl.search().where("jerk_score > 1.2875 AND success = false")` pulls the rough, failed frames out of 27.6M rows in milliseconds, ready to become a fine-tuning set or a labeling queue. diff --git a/docs/web/integrations/reranking/cohere.mdx b/docs/web/integrations/reranking/cohere.mdx index e484f3ee0..26b98a7c8 100644 --- a/docs/web/integrations/reranking/cohere.mdx +++ b/docs/web/integrations/reranking/cohere.mdx @@ -5,7 +5,7 @@ description: Integrate Cohere's powerful reranking API with LanceDB for enhanced --- -import { PyRerankingCohereUsage } from '/snippets/integrations.mdx'; +import { PyRerankingCohereUsage } from '/snippets/provider_examples.mdx'; # Cohere Reranker @@ -18,6 +18,8 @@ This reranker uses the [Cohere](https://cohere.ai/) API to rerank the search res pip install cohere ``` +Replace the credential placeholders with your provider credentials. In application code, load credentials from environment variables or a secrets manager. + {PyRerankingCohereUsage} diff --git a/docs/web/integrations/reranking/jina.mdx b/docs/web/integrations/reranking/jina.mdx index 4624149d2..450a18ecf 100644 --- a/docs/web/integrations/reranking/jina.mdx +++ b/docs/web/integrations/reranking/jina.mdx @@ -5,7 +5,7 @@ description: Integrate Jina's multilingual reranking API with LanceDB for improv --- -import { PyRerankingJinaUsage } from '/snippets/integrations.mdx'; +import { PyRerankingJinaUsage } from '/snippets/provider_examples.mdx'; # Jina Reranker @@ -15,6 +15,8 @@ This reranker uses the [Jina](https://jina.ai/reranker/) API to rerank the searc > **Note:** Supported query types – Hybrid, Vector, and FTS. +Replace the credential placeholders with your provider credentials. In application code, load credentials from environment variables or a secrets manager. + {PyRerankingJinaUsage} diff --git a/docs/web/integrations/reranking/openai.mdx b/docs/web/integrations/reranking/openai.mdx index 413f95e67..22d2af2cc 100644 --- a/docs/web/integrations/reranking/openai.mdx +++ b/docs/web/integrations/reranking/openai.mdx @@ -5,7 +5,7 @@ description: Explore experimental search reranking using OpenAI's GPT models in --- -import { PyRerankingOpenaiUsage } from '/snippets/integrations.mdx'; +import { PyRerankingOpenaiUsage } from '/snippets/provider_examples.mdx'; # OpenAI Reranker (Experimental) @@ -14,6 +14,9 @@ This reranker uses OpenAI chat model to rerank the search results. You can use t > **Warning:** This reranker is experimental. OpenAI does not have a dedicated reranking model, so it uses a chat model under the hood. +Replace `your-api-key` with your provider API key. In application code, load +credentials from environment variables or a secrets manager. + {PyRerankingOpenaiUsage} diff --git a/docs/web/integrations/reranking/voyageai.mdx b/docs/web/integrations/reranking/voyageai.mdx index 62e9a16da..80a666f56 100644 --- a/docs/web/integrations/reranking/voyageai.mdx +++ b/docs/web/integrations/reranking/voyageai.mdx @@ -5,7 +5,7 @@ description: Integrate VoyageAI's cutting-edge reranking models with LanceDB. Fe --- -import { PyRerankingVoyageaiUsage } from '/snippets/integrations.mdx'; +import { PyRerankingVoyageaiUsage } from '/snippets/provider_examples.mdx'; # VoyageAI Reranker @@ -17,6 +17,9 @@ This reranker uses the [VoyageAI](https://docs.voyageai.com/docs/) API to rerank > **Note:** Supported query types – Hybrid, Vector, and FTS. +Replace `your-api-key` with your provider API key. In application code, load +credentials from environment variables or a secrets manager. + {PyRerankingVoyageaiUsage} diff --git a/docs/web/integrations/reranking/watsonx.mdx b/docs/web/integrations/reranking/watsonx.mdx index 775af4f15..5e4c6322c 100644 --- a/docs/web/integrations/reranking/watsonx.mdx +++ b/docs/web/integrations/reranking/watsonx.mdx @@ -5,7 +5,7 @@ description: Rerank LanceDB search results with the IBM watsonx.ai text rerank A --- -import { PyRerankingWatsonxUsage } from '/snippets/integrations.mdx'; +import { PyRerankingWatsonxUsage } from '/snippets/provider_examples.mdx'; # Watsonx Reranker @@ -17,6 +17,8 @@ This reranker uses the [IBM watsonx.ai](https://cloud.ibm.com/docs/apis/watsonx- pip install ibm-watsonx-ai ``` +Replace the credential placeholders with your provider credentials. In application code, load credentials from environment variables or a secrets manager. + {PyRerankingWatsonxUsage} diff --git a/docs/web/maintenance/background-jobs.mdx b/docs/web/maintenance/background-jobs.mdx index ba88b7137..151828e5d 100644 --- a/docs/web/maintenance/background-jobs.mdx +++ b/docs/web/maintenance/background-jobs.mdx @@ -1,22 +1,29 @@ --- title: "Background jobs" sidebarTitle: "Background Jobs" -description: "Jobs a deployment runs on its own — visible in SHOW JOBS, not started by a call you make." +description: "Understand the Enterprise background jobs that maintain tables, indexes and caches." icon: gears keywords: ["background jobs", "job type", "wal", "spfresh", "prewarm", "archive"] +availability: + oss: unavailable + enterprise: available + summary: >- + LanceDB Enterprise schedules background maintenance jobs according to your + cluster configuration. --- -Enterprise - -Every job documented elsewhere in these docs -- [Build & Repair](/indexing/reindexing), +The jobs documented elsewhere in these docs -- [Build & Repair](/indexing/reindexing), [Compaction](/maintenance/compaction), [Cleanup](/maintenance/cleanup), -[Column Refresh](/feature-engineering/refresh-column) -- is one you cause, -directly or by a schedule you set. This page is the rest: jobs the deployment -runs to keep itself correct and fast, which you may see named in -[`SHOW JOBS`](/sql/ddl/job#show) without having asked for any of them. +[Column Refresh](/feature-engineering/refresh-column) -- start from a call you +make or, for compaction, cleanup and index updates, from the deployment's own +maintenance. This page is the rest: jobs the deployment runs to keep itself +correct and fast, which you may see named in [`SHOW JOBS`](/sql/ddl/job#show) +without having asked for any of them. -There is no action to take on these beyond recognizing the name. None of them -has a parameter you set or a call you make. +These jobs are configured by the deployment's operators rather than through +the table SDK. If a maintenance job repeatedly fails or stops making progress, +share its job ID and errors with your Enterprise support team. Operators can +also start cache-warming jobs through the admin API. ## Write-ahead log maintenance {#wal} @@ -41,7 +48,7 @@ log from growing without bound: | --- | --- | | `prewarm_page_cache` | Loads table data into a query node's page cache ahead of a request needing it. | | `prewarm_index_cache` | The same, for index structures. | -| `startup_prewarm_idx` | Runs cache prewarming as part of a query node's own startup, rather than on a schedule. | +| `startup_prewarm_idx` | Runs cache prewarming as part of a plan executor's own startup, rather than on a schedule, when the deployment enables it. | ## Registry housekeeping {#registry} @@ -51,15 +58,16 @@ log from growing without bound: | `job_events_export` | Copies terminal jobs' event histories into a curated system table for metering. Registry-scoped, like the archive sweep. | These two are why an old job you completed weeks ago no longer shows in -`SHOW JOBS` -- it was archived, not deleted; its events survive in the exported -table. +`SHOW JOBS` -- it was archived, not deleted. Where the deployment enables the +export, its events also survive in the exported table. ## Object teardown {#teardown} | Job type | What it does | | --- | --- | -| `drop_object` | Deletes a dropped catalog object's content, then clears it from the database's cleanup backlog. Covers namespaces, views, and functions -- everything [`drop_table`](/management/table#drop) does not, since that one fences a directory with its own marker instead. | +| `drop_object` | Deletes a dropped catalog object's content, then clears it from the database's cleanup backlog. Whether a dropped table runs as this job or as [`drop_table`](/management/table#drop), which fences the table's directory with its own marker instead, depends on how the deployment stores its catalog. | -You trigger this indirectly, by dropping a [namespace](/management/namespace), -[view](/sql/ddl/view#drop), or [function](/management/function#drop) -- the teardown -itself runs as this job rather than inline with the call. +You trigger it indirectly, by dropping a table, a [view](/sql/ddl/view#drop) or +a [namespace](/management/namespace); the teardown runs as this job rather than +inline with the call. +{/* Draft for the Functions release: Dropping a [function](/management/function#drop) starts it the same way. */} diff --git a/docs/web/maintenance/cleanup.mdx b/docs/web/maintenance/cleanup.mdx index 71447a7f7..60a2f6bb0 100644 --- a/docs/web/maintenance/cleanup.mdx +++ b/docs/web/maintenance/cleanup.mdx @@ -4,6 +4,11 @@ sidebarTitle: "Cleanup" description: "Pruning old versions and the files only they referenced." icon: broom keywords: ["cleanup", "retention", "prune", "delete_unverified", "vacuum"] +availability: + oss: available + enterprise: available + summary: >- + Both offerings support cleanup. Embedded `optimize()` uses a retention window of 7 days by default; Enterprise uses configured background cleanup and retention, and does not support `optimize()`. --- Every write keeps the version before it, which is what makes @@ -14,7 +19,8 @@ those versions referenced. **Enterprise job type:** `cleanup`. Distinct from `compact` -- compaction merges files, cleanup removes them. `optimize()` in OSS runs both in one call; -Enterprise schedules them as separate jobs. +Enterprise schedules them as separate jobs, and runs cleanup only when the +deployment's configuration enables it. ## Retention {#retention} @@ -26,17 +32,43 @@ table.optimize(cleanup_older_than=timedelta(days=30)) ``` `cleanup_older_than` defaults to **7 days**. Versions older than that are -removed, along with the files only they referenced. +removed, along with the files only they referenced. In LanceDB Enterprise, the +retention period comes from its configuration instead. -The latest version is never removed, whatever you pass -- `timedelta(0)` prunes -everything except it. +The latest version is never removed, whatever you pass. `timedelta(0)` makes +every older version eligible for removal, but cleanup still keeps tagged +versions and the files branches still read, as the warning below describes. -Pruning is what makes [time travel](/management/table#versioning) finite. A -version outside the retention window is gone, and a tag or branch pointing at -it no longer resolves. +Pruning is what makes [time travel](/management/table#versioning) finite: an +eligible version outside the retention window is gone. Cleanup keeps a version, +whatever its age, when it is: -If you rely on reading old versions, set the window to cover the range you +- the **latest** version; +- a **tagged** version. + +Cleanup also keeps the files a **branch** still reads, and the version the +branch was forked from while the branch's remaining versions read the parent's +files. That does not keep the fork for good: once the branch has replaced the +data it inherited and its own older versions are pruned, cleanup prunes the +fork version like any other. To keep a version recoverable, tag it, subject to +the policy below, or keep the window wide enough to cover it. + +Under the default policy, a tag can also stop cleanup. When a tagged version is +older than the window and is not the latest, cleanup refuses to run: it raises +an error naming the tag and prunes nothing. To let it proceed, move the tag to a +version inside the window, delete the tag, or widen the window. + +A tag on the latest version does not stop cleanup. But `optimize()` compacts +before it prunes, and compacting commits a new version whenever it merges +files, so a tag on what was the latest version can become an older one by the +time cleanup runs. `optimize()` then raises after compacting, and its index +update does not run. + +A deployment's background cleanup follows the same rules. Its own background +jobs, such as compaction and index updates, also commit new versions, so a tag +on what is now the latest version can become an older one without a call from +you. If you rely on reading old versions, set the window to cover the range you need before running this, not after. @@ -46,8 +78,8 @@ Compaction writes new, merged files before the old ones are removed, so disk usage can rise before it falls. Space is reclaimed only when cleanup prunes the versions that referenced the superseded files. -If you need to reclaim space aggressively, shorten the retention window rather -than waiting for the default: +If you need to reclaim space aggressively in the embedded library, shorten the +retention window rather than waiting for the default: ```python table.optimize(cleanup_older_than=timedelta(days=1)) @@ -63,6 +95,9 @@ old, because it cannot tell them apart from a live write. table.optimize(delete_unverified=True) ``` +This option belongs to `optimize()` in the embedded library. A deployment's +cleanup jobs never set it. + `delete_unverified=True` removes them regardless of age. Only set it when you can guarantee **no other process is writing to the table** -- otherwise it can diff --git a/docs/web/maintenance/compaction.mdx b/docs/web/maintenance/compaction.mdx index b6697921c..757a344fc 100644 --- a/docs/web/maintenance/compaction.mdx +++ b/docs/web/maintenance/compaction.mdx @@ -3,18 +3,27 @@ title: "Compaction and cleanup" sidebarTitle: "Compaction" description: "Merge small files, prune old versions, and fold new rows into existing indexes." icon: broom +availability: + oss: available + enterprise: available + summary: >- + Tables are compacted in the embedded library and in LanceDB Enterprise. In the + embedded library you call `optimize()`; a deployment compacts tables in the + background, and `optimize()` is not available there. --- import { PyStorageRebuildVectorIndex } from '/snippets/doc_storage_examples.mdx'; Every write commits a new table version, and versions accumulate: small files from frequent appends, superseded files from updates and deletes, and rows added -since an index was built. `optimize()` addresses all three. +since an index was built. In the embedded library, `optimize()` addresses all +three; a deployment handles them in the background and does not support +`optimize()`. -It is modelled after `VACUUM` in PostgreSQL — routine housekeeping that a table -needs periodically rather than a repair for something broken. +Run optimization as regular table maintenance, particularly after many small +writes or a large ingest. -## What it does {#what-it-does} +## Optimize a table {#what-it-does} ```python table.optimize() @@ -60,7 +69,9 @@ and can help after a large ingest that changed the data distribution. ## When to run it {#when} -There is no background compactor: `optimize()` runs when you call it. +In the embedded library there is no background compactor: `optimize()` runs +when you call it. A deployment compacts a table automatically once enough small +fragments accumulate, unless its operator turns this off. Frequent small appends are what make it necessary, because each one commits files. A table written in a few large batches needs it rarely; one taking @@ -92,4 +103,4 @@ table.optimize() # 7-day default retention table.optimize(cleanup_older_than=timedelta(days=1)) # reclaim space sooner ``` -The best practice is to run `optimize()` after large writes or on a schedule. It also bundles incremental index updates — see [Build & Repair](/indexing/reindexing) for the breakdown. Updates also move rows out of the vector index, so they remain searchable but unindexed — rebuild the index after large update batches. +The best practice is to run `optimize()` after large writes or on a schedule. It also bundles incremental index updates. See [Build & Repair](/indexing/reindexing) for the breakdown. Updates also move rows out of the vector index, so they remain searchable but unindexed. Update or rebuild the index after large update batches. diff --git a/docs/web/management/database.mdx b/docs/web/management/database.mdx index 1f77e5b10..a4b76f195 100644 --- a/docs/web/management/database.mdx +++ b/docs/web/management/database.mdx @@ -1,97 +1,73 @@ --- title: "Database" sidebarTitle: "Database" -description: "Connecting to LanceDB: the URI forms, their options, and what each one changes." +description: "Connect to LanceDB and configure storage, credentials and read consistency." icon: database keywords: ["connect", "database", "uri", "api key", "storage options", "consistency"] +availability: + oss: available + enterprise: available + summary: >- + OSS opens tables in a local directory or object storage. Enterprise connects + to remote compute with a database URI and credentials. --- -A database is the root of a [catalog](/basics/catalog). A connection opens -exactly one, and every other object is reached through it. +import { PyConnectEnterprisePlaceholders, TsConnectEnterprisePlaceholders } from '/snippets/connection.mdx'; -Connecting is the one decision that reaches furthest: the URI you pass decides -where queries execute, which features exist, and what a table handle actually -is. +A database connection provides access to tables and namespaces in a +[catalog](/basics/catalog). Use it to create and open tables, list catalog +objects, and configure how your application reaches the data. -## The three URI forms {#uris} +## Connection URIs {#uris} + +| URI | Offering | Where queries run | +| --- | --- | --- | +| `./.lancedb` | OSS, with local storage | In your application | +| `s3://bucket/path`, `gs://bucket/path`, `az://container/path` | OSS, with object storage | In your application | +| `db://my-database` | Enterprise | On remote compute | + +[Connect to LanceDB](/basics/connect) has examples for each connection type. +A connection to object storage still uses the embedded library: your process +reads and writes the bucket directly. Enterprise runs operations remotely and +manages the supporting compute and maintenance. + +## Connect to Enterprise {#deployment} + +Replace the placeholders with the database URI, key, region and endpoint +provided by your administrator. + + + + {PyConnectEnterprisePlaceholders} + + + { "import * as lancedb from \"@lancedb/lancedb\";\n\n" } + {TsConnectEnterprisePlaceholders} + + + +| Python option | TypeScript option | Purpose | +| --- | --- | --- | +| `api_key` | `apiKey` | Authenticate the connection | +| `region` | `region` | Select the database's region | +| `host_override` | `hostOverride` | Set the Enterprise endpoint | +| `client_config` | `clientConfig` | Configure retries, timeouts and connection pooling. A retry can [repeat an append](/ingestion/insert#performance). | + +Use your deployment's region rather than relying on the `us-east-1` default. +For application code, load credentials from environment variables or a secrets +manager. See [Authentication](/enterprise/authentication) for OAuth and +[SQL](/basics/sql) for the separate SQL connection settings. + +## Storage options {#storage-options} + +For OSS tables in object storage, `storage_options` configures credentials, +endpoints and options for new tables. This example enables stable row IDs for +tables created through the connection: ```python Python icon="python" import lancedb -db = lancedb.connect("./.lancedb") # local directory -db = lancedb.connect("s3://bucket/path") # object storage -db = lancedb.connect("db://my-database", api_key="...") # a deployment -``` - -```typescript TypeScript icon="square-js" -import * as lancedb from "@lancedb/lancedb"; - -const db = await lancedb.connect("./.lancedb"); -const db = await lancedb.connect("s3://bucket/path"); -const db = await lancedb.connect("db://my-database", { apiKey: "..." }); -``` - - -The first two are **embedded**: your process opens Lance tables directly from -storage and does the work itself. The third talks to a **deployment**. - -| | Embedded | Deployment | -| --- | --- | --- | -| Where queries run | in your process | on the server | -| Table handle | `LanceTable` | `RemoteTable` | -| [SQL](/basics/sql) | -- | yes | -| Server-side [jobs](/basics/jobs) | -- | yes | -| [Functions](/management/function) | -- | yes | -| [Authorization](/governance/authorization) | storage permissions | granted per object | - -![](/static/assets/images/overview/understanding-tables.png) - -Both handles expose the same table API -- create, open, mutate, query -- so -application code moves between them by changing the URI. What differs is where -resolution and execution happen, and therefore which operations exist at all. - -## Connecting to a deployment {#deployment} - -Enterprise - - -```python Python icon="python" -db = lancedb.connect( - "db://my-database", - api_key="...", - region="us-east-1", -) -``` - -```typescript TypeScript icon="square-js" -const db = await lancedb.connect("db://my-database", { - apiKey: "...", - region: "us-east-1", -}); -``` - - -| Option | Meaning | -| --- | --- | -| `api_key` / `apiKey` | authenticates the connection -- see [Authentication](/enterprise/authentication) | -| `region` | which region hosts the database. Defaults to `us-east-1` | -| `host_override` / `hostOverride` | point at a specific endpoint instead of the regional one | -| `sql_host_override` | the same, for the [SQL endpoint](/sql#connecting). Python only | -| `client_config` / `clientConfig` | retries, timeouts, and connection pooling | - -An API key belongs in the environment, not in source. `region` is easy to get -wrong silently -- the default is `us-east-1` whether or not that is where your -database lives, and a wrong region fails as a connection error rather than a -helpful one. - -## Storage options {#storage-options} - -`storage_options` configures the underlying object store -- credentials, -endpoints, and defaults for tables created through this connection: - - -```python Python icon="python" db = lancedb.connect( "s3://bucket/path", storage_options={"new_table_enable_stable_row_ids": "true"}, @@ -99,95 +75,89 @@ db = lancedb.connect( ``` ```typescript TypeScript icon="square-js" +import * as lancedb from "@lancedb/lancedb"; + const db = await lancedb.connect("s3://bucket/path", { storageOptions: { new_table_enable_stable_row_ids: "true" }, }); ``` - -`new_table_enable_stable_row_ids` is worth setting deliberately: it gives stable -row ids to the tables this connection creates, and no SDK call adds them to an -existing table. A `db://` connection does not send it; a deployment decides -stable row ids itself (see -[Against a deployment](/feature-engineering/materialized-views#availability)). - +Set stable row IDs before creating a table that needs them, such as an ordinary +materialized-view source in Python SDK 0.40.0. The option does not change tables +that already exist. Enterprise storage settings are managed on the server. -See [Storage](/deployment/storage) for the full set, including per-cloud credentials. +See [Storage](/deployment/storage) for provider credentials and other options. ## Read consistency {#consistency} -`read_consistency_interval` controls how often a connection checks for versions -committed by other writers. +For OSS connections, `read_consistency_interval` controls how often reads check +for table versions committed by other writers. For example, check at most once +every five seconds: ```python Python icon="python" from datetime import timedelta +import lancedb -db = lancedb.connect(uri, read_consistency_interval=timedelta(seconds=5)) +db = lancedb.connect( + "./.lancedb", read_consistency_interval=timedelta(seconds=5) +) ``` ```typescript TypeScript icon="square-js" -const db = await lancedb.connect(uri, { readConsistencyInterval: 5 }); +import * as lancedb from "@lancedb/lancedb"; + +const db = await lancedb.connect("./.lancedb", { readConsistencyInterval: 5 }); ``` -The default reads a snapshot taken when the table was opened, so a long-lived -connection **will not see other processes' writes** until it reopens the table. -That is the right default for a batch job and the wrong one for a service. +By default, an OSS table handle does not automatically pick up another +process's writes. Reopen the table, check out its latest version, or configure +a consistency interval when you need those updates. A zero interval checks on +every read, with the cost of a metadata check. -Set it when several processes write the same table and readers must observe -those writes. The cost is a metadata check on that interval. See -[Consistency](/tables/consistency). +Enterprise manages read freshness on the server. See +[Consistency](/tables/consistency) for its behavior and the distinction between +seeing your own writes and seeing another client's writes. -## Database-level operations {#operations} +## Database operations {#operations} -| | Python | TypeScript | SQL | -| --- | --- | --- | --- | -| list tables | `table_names`, `list_tables` | `tableNames`, `listTables` | -- | -| the URI it opened | `uri` | `display()` | -- | -| drop every table | `drop_all_tables` | `dropAllTables` | -- | -| drop the database | `drop_database` | -- | -- | -| close | -- | `close`, `isOpen` | -- | +The connection provides methods to create, open, list and drop tables. - -```python Python icon="python" -db.table_names() -tbl = db.open_table("documents") -db.drop_table("documents") -``` +| Operation | Python | TypeScript | +| --- | --- | --- | +| Create a table | `create_table` | `createTable` | +| Open a table | `open_table` | `openTable` | +| List tables | `table_names`, `list_tables` | `tableNames`, `listTables` | +| Drop a table | `drop_table` | `dropTable` | -```typescript TypeScript icon="square-js" -await db.tableNames(); -const tbl = await db.openTable("documents"); -await db.dropTable("documents"); -``` - -```sql SQL -DROP TABLE documents; -``` - - -Each takes a `namespace_path` to act inside a [namespace](/management/namespace) rather -than at the root. +Use the namespace and pagination options documented in +[Namespaces](/management/namespace) when working outside the root namespace +or listing a large catalog. [Tables](/management/table) covers table schemas, +versions and lifecycle operations. -`drop_database()` removes every table in the database. There is no confirmation -and no undo -- table [versioning](/management/table#versioning) does not survive the table -being dropped. +Dropping a table removes its history as well as its current data. You cannot +recover a dropped table through table versioning. The OSS `drop_database()` +and `drop_all_tables()` operations remove all tables without a confirmation prompt. ## Async connections {#async} -`connect_async` returns an `AsyncConnection` with the same surface as -coroutines. The TypeScript SDK is async throughout, so its `connect` already is -one. +Python's `connect_async` returns an `AsyncConnection`. Await its operations +when working in an asynchronous application: ```python -db = await lancedb.connect_async("db://my-database", api_key="...") -tbl = await db.open_table("documents") +import lancedb + +db = await lancedb.connect_async("./.lancedb") +table = await db.open_table("documents") ``` -This is a different axis from the `_async` operations that return a -[job](/basics/jobs): `connect_async` is about not blocking your event loop, -while `create_index_async` is about work that outlives the call. +TypeScript's connection and table operations already return promises. + +An async connection and a background job serve different purposes. Async APIs +let other tasks run while your application waits for an operation. A +[job](/basics/jobs) represents work that can continue on the server after the +submission call returns. diff --git a/docs/web/management/function.mdx b/docs/web/management/function.mdx index 40e5bcd00..5734d2545 100644 --- a/docs/web/management/function.mdx +++ b/docs/web/management/function.mdx @@ -4,27 +4,31 @@ sidebarTitle: "Function" description: "Register a Python function once and apply it as a column, for work SQL cannot express." icon: code keywords: ["udf", "functions", "create_function", "embedding", "feature engineering"] +availability: + oss: unavailable + enterprise: available + summary: >- + Enterprise supports Functions when enabled for your Enterprise cluster; the embedded library cannot register or run them. --- import { PyFunctionDefine as FunctionDefine, PyFunctionLocalCall as FunctionLocalCall, - PyFunctionConnect as FunctionConnect, PyFunctionCreateTable as FunctionCreateTable, PyFunctionRegister as FunctionRegister, PyFunctionApply as FunctionApply, PyFunctionRefresh as FunctionRefresh, PyFunctionInspect as FunctionInspect, } from '/snippets/function.mdx'; - -Enterprise +import { PyConnectEnterprisePlaceholders as FunctionConnect } from '/snippets/connection.mdx'; Some columns cannot be written as a SQL expression: an embedding, a model call, an image decode. A function is a Python callable registered with the database, versioned, and then applied as a column — the code runs where the data is rather than pulling rows to your process. -Registering and running a function needs a LanceDB Enterprise connection. On a +Registering and running a function needs a LanceDB Enterprise deployment with +Functions enabled, which they are not by default. On a local connection `create_function` raises `NotImplementedError`; to derive a column locally, declare a [SQL computed column](/feature-engineering/add-computed-column) instead. @@ -33,6 +37,10 @@ column locally, declare a In SQL, a function is registered with [`CREATE FUNCTION`](/sql/ddl/function#create) and listed with [`SHOW FUNCTIONS`](/sql/ddl/function#show). +Functions are the way to fill columns with Python in LanceDB Enterprise. If you +already use [Geneva](/geneva), it remains available; see +[Geneva and Functions](/geneva#functions) for how its concepts map onto these. + The steps below take a column of numbers to a filled column of their doubles. ## Define a function {#define} @@ -62,9 +70,9 @@ sibling columns, so a null field is a value rather than a missing row. ## Connect and create a table {#table} -The remaining steps run against your Enterprise deployment. This connection -reads its settings from environment variables, so no credential appears in the -code: +The remaining steps run against your Enterprise deployment. Replace the database +URI, API key, region and endpoint below with your deployment details. In application +code, load credentials from environment variables or a secrets manager: {FunctionConnect} @@ -75,7 +83,7 @@ A nullable column requires a nullable parameter, such as `Optional[float]`; a non-nullable column can bind to either. This example uses `value: float`, so the table declares `value` non-nullable. -Computed-column inputs depend on the deployment version. Where supported, +Computed-column inputs depend on the Enterprise server version. Where supported, they bind to non-nullable parameters even though the physical column is nullable: refresh skips rows whose computed input is still null. Check your deployment's support before using another computed column as an input. @@ -212,6 +220,10 @@ the name was already gone. Deleting the object's content can finish after the call returns, as a [background job](/maintenance/background-jobs). `drop_function_async` returns that job with the result, so you can wait for it. +A deployment refuses the drop unless it also allows changes to a function's +lifecycle, which is a separate setting, off by default even where Functions are +enabled. + ## What to read next {#next} diff --git a/docs/web/management/index.mdx b/docs/web/management/index.mdx index 1d4d4eeba..b6bc07e88 100644 --- a/docs/web/management/index.mdx +++ b/docs/web/management/index.mdx @@ -6,17 +6,17 @@ icon: folder-tree keywords: ["ddl", "create", "drop", "list", "catalog", "management"] --- -Every [catalog object](/basics/catalog) is created, listed, and dropped the same -few ways. This section is the reference for those operations; the pages linked -from it cover what each object is for. +Use the SDKs or SQL to manage [catalog objects](/basics/catalog), including +tables, namespaces and views. The tables below provide a quick reference to +the available methods; each object's guide covers its setup and behavior. -Operations that can take a while have an `_async` form returning a -[job](/basics/jobs). +Some long-running operations also provide a submission method that returns a +[job](/basics/jobs). See the operation's guide for how to track completion. -## The surface at a glance {#surface} +## Management APIs {#surface} -SQL requires a deployment -- a local database has no engine to plan a statement. +SQL is available through the Enterprise Flight SQL endpoint. See [SQL](/basics/sql#sql-is-remote). @@ -59,8 +59,8 @@ See [Tables](/management/table) for creating one and shaping its schema. | drop | `drop_table` | `dropTable` | `DROP TABLE` | -`list_materialized_views` reads every table's schema to find them, so it costs -one open per table. `SHOW MATERIALIZED VIEWS` asks the server instead. +Local `list_materialized_views` reads table schemas to find views. On +Enterprise, it requests the server's view listing. See [Materialized views](/management/materialized-view) for creating one, and @@ -71,13 +71,17 @@ it current. | | Python | TypeScript | SQL | | --- | --- | --- | --- | -| create | -- | -- | `CREATE VIEW` | -| drop | -- | -- | `DROP VIEW` | +| create | `create_view` | `createView` | `CREATE VIEW` | +| list | `list_views` | `listViews` | -- | +| drop | `drop_view`, `drop_view_async` | `dropView`, `dropViewAsync` | `DROP VIEW` | -Enterprise Views exist only in SQL -- there is no SDK -method for them, because the thing a view stores *is* a SQL statement. See +Enterprise Views exist only on an Enterprise cluster that +enables them; the SDK methods send the view's SQL query to the deployment. See [Views](/management/view). +{/* +Draft for the Functions release. + ### Functions {#functions} | | Python | TypeScript | SQL | @@ -87,9 +91,11 @@ method for them, because the thing a view stores *is* a SQL statement. See | list | `list_functions` | -- | `SHOW FUNCTIONS` | | drop | `drop_function` | -- | -- | -Enterprise A function runs on the server, so there is -no local form. The TypeScript SDK has no function methods; use Python or SQL. +Enterprise A function runs on the server, and only on +a deployment that enables functions, so there is no local form. The TypeScript +SDK has no function methods; use Python or SQL. See [Functions](/management/function). +*/} ### Jobs {#jobs} @@ -119,7 +125,7 @@ names it. For a multi-column or named index, use the SDK. Indexes are a performance decision rather than a structural one, so they are covered under [Maintenance](/indexing). -## Namespaces qualify everything {#namespace-paths} +## Work within a namespace {#namespace-paths} Most connection-level operations take a `namespace_path` (`namespacePath`) to act inside a namespace rather than at the database root: @@ -131,8 +137,8 @@ tbl = db.open_table("clicks", namespace_path=["prod", "events"]) ``` ```typescript TypeScript icon="square-js" -await db.tableNames({ namespacePath: ["prod", "events"] }); -const tbl = await db.openTable("clicks", { namespacePath: ["prod", "events"] }); +await db.tableNames(["prod", "events"]); +const tbl = await db.openTable("clicks", ["prod", "events"]); ``` ```sql SQL @@ -142,7 +148,7 @@ SELECT * FROM prod.events.clicks In SQL the path is written as dotted identifiers instead. -## Dropping is not undoable {#dropping} +## Delete catalog objects {#dropping} `drop_table` removes the table and every version of it. Table diff --git a/docs/web/management/materialized-view.mdx b/docs/web/management/materialized-view.mdx index cbdb703b1..1475bd93a 100644 --- a/docs/web/management/materialized-view.mdx +++ b/docs/web/management/materialized-view.mdx @@ -4,6 +4,11 @@ sidebarTitle: "Materialized View" description: "Create, open, list and drop a table whose contents are defined by a query." icon: table keywords: ["materialized view", "create_materialized_view", "catalog", "ddl"] +availability: + oss: available + enterprise: available + summary: >- + Both offerings support materialized views. See [Enterprise compatibility](#deployment) for connection and server requirements. --- A materialized view is a table whose contents are defined by a query over @@ -20,7 +25,7 @@ This page covers creating and managing one. Keeping it current is In SQL, see [`REFRESH MATERIALIZED VIEW`](/sql/dml/refresh#refresh-materialized-view) and [`SHOW MATERIALIZED VIEWS`](/sql/ddl/materialized-view#show). -## Creating one {#create} +## Create a materialized view {#create} ```python Python icon="python" @@ -28,24 +33,24 @@ view = db.create_materialized_view( "recent_docs", source="documents", select=["id", "title", "embedding"], - where="created_at > '2026-01-01'", + where="created_at > TIMESTAMP '2026-01-01 00:00:00'", ) ``` ```typescript TypeScript icon="square-js" const view = await db.createMaterializedView("recent_docs", "documents", { select: ["id", "title", "embedding"], - where: "created_at > '2026-01-01'", + where: "created_at > TIMESTAMP '2026-01-01 00:00:00'", }); ``` -`source` names the table it derives from. A local source does not need stable -row ids or any other storage option; a deployment may require stable row ids -(see [Against a deployment](#deployment)). `select`, `where` and `limit` record -the query -- there is no separate definition step. +`source` names the table the view reads. With Python SDK 0.40.0, create that +table with stable row IDs before creating the view. The +[complete example](/feature-engineering/materialized-views#create-and-refresh) +shows the setup. `select`, `where` and `limit` define the query. -## Opening, listing and dropping {#manage} +## Open, list and drop views {#manage} | | Python | TypeScript | SQL | | --- | --- | --- | --- | @@ -56,34 +61,30 @@ the query -- there is no separate definition step. | drop | `drop_table` | `dropTable` | `DROP TABLE` | -`list_materialized_views` finds views by reading every table's schema, so it -costs one open per table. `SHOW MATERIALIZED VIEWS` asks the server instead and -is cheaper on a catalog with many tables. +On a local database, `list_materialized_views` reads table schemas to find +views. On Enterprise, it asks the server for the view listing, as +`SHOW MATERIALIZED VIEWS` does. A view is dropped with `drop_table`, because a materialized view **is** a table. That also means dropping one removes stored rows -- unlike dropping a [view](/management/view), which removes only a saved query. -## Working with the view {#working} +## Read the view and its definition {#working} `view.table` is the underlying table, for querying, indexing, or search. -`view.definition` is the recorded query -- select list, predicate, limit --- which is how you check what a view contains without inferring it from the -rows. Both are properties; on an async view, the definition is read with +`view.definition` contains the query's select list, predicate and limit. Both are properties; on an async view, the definition is read with `await view.definition()`. -## Against a deployment {#deployment} +## Enterprise compatibility {#deployment} -Enterprise `create_materialized_view` works on a -`db://` connection as well as a local one. There the SDK creates and refreshes -the view through SQL, and the work runs on the cluster rather than in your -process. Materialized views are off by default on a deployment and have to be -enabled there, and a deployment may require a view's source to have stable row -ids; see -[Against a deployment](/feature-engineering/materialized-views#availability). +Enterprise supports creating and refreshing materialized views on remote +compute. Use the SDK version supported by your deployment, or the +[SQL interface](/sql/ddl/materialized-view). See +[Enterprise refreshes](/feature-engineering/materialized-views#availability) +for compatibility considerations when upgrading an existing pipeline. -## What to read next {#next} +## Related guides {#next} diff --git a/docs/web/management/namespace.mdx b/docs/web/management/namespace.mdx index f1310b6da..a3485f46d 100644 --- a/docs/web/management/namespace.mdx +++ b/docs/web/management/namespace.mdx @@ -4,6 +4,12 @@ sidebarTitle: "Namespace" description: "Grouping tables into a hierarchy, and addressing them through it." icon: folder-tree keywords: ["namespace", "catalog", "hierarchy", "directory namespace", "rest namespace"] +availability: + oss: available + enterprise: available + summary: >- + Namespaces work in the embedded library, through directory or REST + catalogs, and in LanceDB Enterprise. --- import { @@ -15,8 +21,10 @@ import { RsNamespaceTableOps, } from '/snippets/connection.mdx'; -Despite its name, LanceDB is not a "database" in the traditional sense -- it is a **Multimodal Lakehouse** that builds on the table abstraction, -similar to many other lakehouse projects. LanceDB exposes a catalog-level abstraction over the Lance table format, via a *namespace spec*. +LanceDB organizes Lance tables through a catalog-level abstraction defined by +the *namespace spec*. LanceDB OSS is an embedded library for accessing those +tables, and LanceDB Enterprise is a multimodal lakehouse that provides shared +compute and catalog services on top of the Lance format. If you're coming from traditional databases or lakehouses, you can think of a namespace as the catalog path that says where a table name lives. Lance provides the **file** and **table** formats to store and manage your data and indexes. @@ -29,7 +37,7 @@ language: Python uses `namespace_path`, Rust uses builder methods like `.namespa TypeScript uses `namespacePath` arguments. -In SQL a namespaced table is addressed as a [dotted path](/sql/dql/select#namespaces). +In LanceDB Enterprise SQL, a namespaced table is addressed as a [dotted path](/sql/dql/select#namespaces). ## Namespace hierarchy {#namespace-hierarchy} @@ -47,14 +55,15 @@ If you want to go deeper, see the Lance format [namespace documentation](https:/ A namespace path is a list of components. For example, `["prod", "search"]` means the `search` namespace inside the `prod` namespace. The empty path, `[]`, means the root namespace. -Each component is a name, not a filesystem path segment. Namespace names can't be empty, and each -component can contain only letters, numbers, underscores, hyphens, and periods. That keeps the same -identifier usable across local directory namespaces and REST namespace identifiers. +Each component is a name, not a filesystem path segment, and can't be empty. Keep each one to +letters, numbers, underscores, hyphens, and periods, so the same identifier is usable everywhere: a +local directory namespace accepts other characters too, but other namespace implementations and +deployments can refuse them. [Naming rules](/basics/naming) has the lengths and the other limits. ## Directory namespaces {#directory-namespaces} The simplest namespace model in LanceDB is a single root namespace, often represented by one -directory: +directory. Namespaces can nest inside it, as in this hierarchy: ```bash ./local_lancedb (root) @@ -64,7 +73,12 @@ directory: └── data (table) ``` -As a user of LanceDB OSS, you might never notice namespaces at first, because LanceDB exposes the single-level hierarchy shown above, with the data stored in the `data/` directory, where the root namespace is implicit. Connecting to this namespace is as simple as connecting to the catalog root: +The hierarchy is logical, not a tree of folders: a directory namespace records it in a +`__manifest` table in the root directory, and stores each table in a directory of its own there. + +As a user of LanceDB OSS, you might never notice namespaces at first, because a plain connection +uses only the root namespace, which is implicit. Connecting to this namespace is as simple as +connecting to the catalog root: ```python Python icon="python" @@ -89,7 +103,7 @@ let db = connect("./local_lancedb").execute().await?; ``` -This creates the default namespace directory (`data/`) under the specified root path. +A table created through this connection is stored directly under that path, as `.lance`. You can also explicitly connect to a namespace using `lancedb.connect_namespace(...)` with the directory namespace implementation: @@ -152,7 +166,7 @@ println!("Created table: {}", table.name()); - For simple use cases in LanceDB OSS, you don't need to go too deep into namespaces. -- To integrate LanceDB with external catalogs and to use it as a true **multimodal lakehouse**, it's useful to understand the different namespace implementations and how to use them in your organization's setup. +- To integrate LanceDB with external catalogs, understand how each namespace implementation resolves table names and locations in your organization's setup. ## Remote or external catalog namespaces {#remote-or-external-catalog-namespaces} @@ -169,9 +183,11 @@ LanceDB Enterprise REST requests use the `x-api-key` header for API-key authenti that route multiple databases through the same endpoint can also use headers such as `x-lancedb-database` or `x-lancedb-database-prefix` for database context. +Replace the catalog URL and credential placeholders below with your values. In +application code, load credentials from environment variables or a secrets manager. + ```python Python icon="python" -import os import lancedb # Remote namespace-backed catalog root (RestNamespace) @@ -180,9 +196,9 @@ db = lancedb.connect_namespace( "rest", { "uri": "https://.internal..com", - "headers.x-api-key": os.environ["API_KEY"], + "headers.x-api-key": "your-api-key", # or: - # "headers.Authorization": f"Bearer {os.environ['REST_AUTH_TOKEN']}", + # "headers.Authorization": "Bearer your-access-token", }, ) ``` @@ -195,9 +211,9 @@ import * as lancedb from "@lancedb/lancedb"; const db = await lancedb.connectNamespace("rest", { uri: "https://.internal..com", headers: { - "x-api-key": process.env.API_KEY ?? "", + "x-api-key": "your-api-key", // or: - // Authorization: `Bearer ${process.env.REST_AUTH_TOKEN}`, + // Authorization: "Bearer your-access-token", }, }); ``` @@ -214,12 +230,12 @@ properties.insert( ); properties.insert( "headers.x-api-key".to_string(), - std::env::var("API_KEY")?, + "your-api-key".to_string(), ); // or: // properties.insert( // "headers.Authorization".to_string(), -// format!("Bearer {}", std::env::var("REST_AUTH_TOKEN")?), +// "Bearer your-access-token".to_string(), // ); let db = lancedb::connect_namespace("rest", properties) .execute() @@ -227,7 +243,8 @@ let db = lancedb::connect_namespace("rest", properties) ``` [LanceDB Enterprise](/enterprise) operates a REST namespace server on top of the Lance format, so any REST client that can speak the REST namespace API -contract can be used to interact with it. For authentication examples in LanceDB Enterprise, visit +contract can be used to interact with it. It does not serve the contract's `exists` routes, so checks such as `namespace_exists` +fail against it. For authentication examples in LanceDB Enterprise, visit the [Namespaces in SDKs](#namespaces-in-lancedb-enterprise) page. ## Best practices {#best-practices} @@ -279,8 +296,8 @@ The SDK methods expose the namespace path in the idiom of each language: Using namespaces is **optional** in LanceDB, and most basic use cases do not require to work with them. -An empty namespace (`[]`), which is the default, means "root namespace", and the data will be stored in -the `data/` directory under the specified root path. +An empty namespace (`[]`), which is the default, means the root namespace, whose tables are stored +directly under the specified root path. ## Namespace management APIs {#namespace-management-apis} @@ -315,12 +332,13 @@ doesn't exist, or contains data: - Drop mode: `fail` reports an error when the namespace doesn't exist, and `skip` treats a missing namespace as a successful no-op. - Drop behavior: `restrict` keeps non-empty namespaces from being dropped, and `cascade` drops child namespaces and tables first. -Namespace path components can't be empty. Each component can contain only letters, numbers, -underscores, hyphens, and periods. +Namespace path components can't be empty, and are best kept to letters, numbers, underscores, +hyphens, and periods, as [above](#namespace-paths-and-names). Listing APIs return the immediate children of the requested namespace path. Use `limit` with the returned `page_token` to page through large catalogs; pass an empty namespace path (`[]`) when you -want to list from the root namespace. +want to list from the root namespace. In Python, a listing returns a response: the names are in its +`namespaces` or `tables`, and `page_token` is set when there is another page. ## Namespaces in LanceDB Enterprise {#namespaces-in-lancedb-enterprise} diff --git a/docs/web/management/table.mdx b/docs/web/management/table.mdx index 7f84b9131..1b8ede070 100644 --- a/docs/web/management/table.mdx +++ b/docs/web/management/table.mdx @@ -4,6 +4,13 @@ sidebarTitle: "Table" description: "The object most operations act on: creating, opening, renaming, cloning and dropping one." icon: table keywords: ["table", "create_table", "open_table", "clone", "rename", "drop"] +availability: + oss: available + enterprise: available + summary: >- + Tables are created, evolved, versioned and dropped in the embedded library + and in LanceDB Enterprise. Renaming, cloning, dropping every table and pruning old + versions differ between the two. --- import { @@ -113,18 +120,20 @@ import { PyQueryToPandasKwargs as QueryToPandasKwargs, } from '/snippets/multimodal.mdx'; -A table is rows and columns, versioned. It is what searches read, what writes -commit to, and what indexes and branches belong to. +A table stores data in rows and columns and keeps a history of committed +versions. Queries read table data, writes update it, and indexes accelerate +searches over its columns. Branches let you make changes separately from the +main table history. -This page covers the table as a catalog object. Getting data into one is -[Ingest](/ingestion); changing its shape is -[Schema evolution](/management/table#schema). +This page covers table management, data types, schema changes and version +history. For loading records, see [Ingestion](/ingestion). For changes to +columns, see [Schema evolution](/management/table#schema). -Tables are created and dropped in SQL with [`CREATE TABLE` and `DROP TABLE`](/sql/ddl/table). +In LanceDB Enterprise, tables are also created and dropped in SQL with [`CREATE TABLE` and `DROP TABLE`](/sql/ddl/table). -## What a table carries {#carries} +## Table structure and metadata {#carries} | | | | --- | --- | @@ -133,7 +142,7 @@ Tables are created and dropped in SQL with [`CREATE TABLE` and `DROP TABLE`](/sq | **Indexes** | vector, full-text and scalar, built over its columns | | **Branches** | isolated lines of change you can [merge or discard](/tables/branching) | -## Creating {#create} +## Create a table {#create} ```python Python icon="python" @@ -156,19 +165,18 @@ frame, or Pydantic models. [Ingesting data](/ingestion) covers each, plus `exist_ok` and `mode="overwrite"` for re-runnable scripts. -An explicit schema is where nullability is decided. Creation is also the only -point at which the SDK can give a table **stable row ids**, which a deployment -may require on a [materialized view](/management/materialized-view) source. See -[Limits](/basics/limits#schema). +An explicit schema is where nullability is decided. In the embedded library, +creation is also the only point at which a table can be given **stable row +ids**; in LanceDB Enterprise, whether a new table gets them is the deployment's +setting. See [Limits](/basics/limits#schema). -## Opening and listing {#open} +## Open and list tables {#open} ```python Python icon="python" tbl = db.open_table("documents") db.table_names() -db.table_exists("documents") ``` ```typescript TypeScript icon="square-js" @@ -179,9 +187,18 @@ await db.tableNames(); Each accepts `namespace_path` to act inside a [namespace](/management/namespace) -rather than at the database root. TypeScript has no `tableExists`. +rather than at the database root. -## Renaming and cloning {#rename-clone} +To find out whether a table exists, open it and handle the error, or look for +its name in a listing. A missing table raises `ValueError` on a local directory +or a deployment, and `lance_namespace.TableNotFoundError` on a connection from +`connect_namespace`. A listing returns one page of names at a time, so check +every page. Python's `table_exists` is no substitute: a local directory or a +deployment raises `NotImplementedError` for it, and on a connection from +`connect_namespace("dir", ...)` or `connect_namespace("rest", ...)` it fails with +an `AttributeError`. TypeScript has no `tableExists`. + +## Rename and clone tables {#rename-clone} ```python db.rename_table("documents", "docs") @@ -194,6 +211,10 @@ db.clone_table( ) ``` +Renaming is not supported on a local directory connection. In LanceDB Enterprise, +whether a table can be renamed or cloned at all depends on how the deployment +stores its catalog. + A **shallow** clone -- the default -- references the source's data files rather than copying them, so it is fast and cheap regardless of table size. It is the practical way to branch an experiment off a known version, or to pin a training @@ -204,18 +225,20 @@ A shallow clone depends on the source's files continuing to exist. If the source is [compacted](/maintenance/compaction) past the retention window, the version the clone points at can be pruned. Local OSS connections support shallow clones only; `is_shallow=False` is not implemented. If the copy must outlive the -source's history, copy its data into an independent table instead. For remote -connections, confirm which clone modes your deployment supports. +source's history, copy its data into an independent table instead. A deployment +that supports cloning also makes shallow clones only, and one whose catalog +does not support it refuses every clone. -**Enterprise job type:** `drop_table`. Fences the table's directory with a -marker and deletes its content in the background -- see -[Background jobs](/maintenance/background-jobs#teardown) for the namespace, -view, and function equivalent. +**Enterprise job type:** on an Enterprise cluster that drops tables in the background, +the teardown runs as a job: `drop_table`, which fences the table's directory +with a marker and deletes its content, or `drop_object`, depending on how the +deployment stores its catalog. See +[Background jobs](/maintenance/background-jobs#teardown). -## Dropping {#drop} +## Drop a table {#drop} ```python Python icon="python" @@ -238,13 +261,16 @@ DROP TABLE documents Dropping removes the table and **every version of it**. [Versioning](/management/table#versioning) does not survive a drop, and neither -`drop_all_tables` nor `drop_database` asks for confirmation. +`drop_all_tables` nor `drop_database` asks for confirmation. Both work only on +a local database; on an Enterprise connection the SDK raises instead. On a large table the drop can take a while, so `drop_table_async` returns a -[job](/basics/jobs) rather than blocking. +[job](/basics/jobs) rather than blocking when the deployment drops tables in the +background. In the embedded library, and on an Enterprise cluster that drops inline, the +returned job has already finished. -## The full surface {#surface} +## Table management APIs {#surface} | | Python | TypeScript | SQL | | --- | --- | --- | --- | @@ -456,7 +482,7 @@ When you call `to_pandas()` on a local LanceDB table that contains Blob API colu - `"bytes"`: eagerly materializes each blob as `bytes`. Use this when you need the raw payload in the DataFrame, for example to decode an image or audio clip in-process. - `"descriptions"`: returns blob descriptors (offsets, sizes, and positions) instead of the data itself. Use this when you want to plan I/O without paying the cost of loading every blob. -`"bytes"` and `"descriptions"` require a filesystem-backed Lance dataset and are not supported on in-memory tables. +`"bytes"` is not supported on in-memory tables. Extra keyword arguments are forwarded to the underlying PyArrow / Lance pandas conversion, so you can also pass options like `split_blocks` or `self_destruct`: @@ -879,8 +905,13 @@ Remove several columns at once for efficiency: - -Dropping columns cannot be undone. Make sure you have backups or are certain before removing columns. + +Dropping a column changes the latest version only. Until +[cleanup](/maintenance/cleanup) prunes the versions from before the drop, +[restoring](#versioning) one of them brings the column and its values back -- +along with the rest of the table as it was then. Once those versions are +pruned, the drop cannot be undone, so check before dropping a column you may +need. ## Versioning {#versioning} @@ -1040,8 +1071,9 @@ Numeric table versions like `v3` or `v17` are precise but hard to remember. Tags let you attach human-readable labels (e.g., `"prod"`, `"baseline"`, `"q3-evaluation"`) to specific versions and check those out by name. They are conceptually similar to git tags, and unlike numeric versions, **tagged versions -are preserved when old versions are pruned** (see the cleanup note at the bottom -of this page). +are never pruned** -- a tagged version that is older than the retention window and +is not the latest stops cleanup instead (see the cleanup note at the bottom of this +page). The tags API supports the standard CRUD operations — create, list, update, delete — plus checking out by tag name. @@ -1137,10 +1169,17 @@ mixing in lower-level Lance file operations. **System Operations** System operations like `optimize()`, index updates, and table compaction also increment table version numbers. -In LanceDB OSS and Enterprise, `optimize()` can prune older versions based on its retention setting (`cleanup_older_than`, 7 days by default), -which is when old-version files are removed and disk space is reclaimed. +In LanceDB OSS, `optimize()` can prune older versions based on its retention setting (`cleanup_older_than`, 7 days by default), +which is when old-version files are removed and disk space is reclaimed. A deployment prunes them in the background +when its configuration enables [cleanup](/maintenance/cleanup); `optimize()` is not available there. -**Tagged versions are exempt from cleanup.** A version with a tag pointing at it is -retained regardless of age, and its files are not removed by `optimize()`. To make -a tagged version eligible for pruning, [delete the tag](#versioning-tag-based-versioning) first. +**Tagged versions are never pruned.** A version with a tag pointing at it is +retained regardless of age. When a tagged version is older than the retention window and +is not the latest, cleanup refuses to run instead of pruning around it: `optimize()` raises +an error naming the tag and prunes nothing. A tag on the latest version does not cause +this, but `optimize()` compacts before it prunes, and compacting commits a new version +whenever it merges files, so a tag on what was the latest version can become an older +one by the time cleanup runs; the error then comes after compacting, and the index +update does not run. To let cleanup proceed, [move the tag](#versioning-tag-based-versioning) +to a version inside the retention window, delete it, or widen the window. diff --git a/docs/web/management/view.mdx b/docs/web/management/view.mdx index bfb3a0543..de6cd38a5 100644 --- a/docs/web/management/view.mdx +++ b/docs/web/management/view.mdx @@ -4,29 +4,31 @@ sidebarTitle: "View" description: "A saved query, evaluated when read. The non-materialized counterpart to a materialized view." icon: eye keywords: ["view", "create view", "sql", "catalog"] +availability: + oss: unavailable + enterprise: available + summary: >- + Enterprise supports views when enabled for your Enterprise cluster; the embedded library cannot create them. --- -Enterprise - -This page describes a LanceDB Enterprise capability. The embedded form -of LanceDB has no server to provide it -- see -[Architecture](/architecture) for what differs between the two. - -A view stores a **query**, not rows. Reading it runs the query against current -data, so a view is never stale -- and never faster than the query it wraps. +A view saves a query that you can refer to by name. Reading the view executes +that query against the source tables, using their read-consistency behavior. +Views do not store a separate copy of the result. It is the counterpart to a [materialized view](/management/materialized-view), which stores rows and is brought up to date by a refresh. Both are [catalog objects](/basics/catalog) -defined by a query; the difference is entirely when that query runs. +defined by a query; choose a materialized view when you need to store and reuse the result. | | View | Materialized view | | --- | --- | --- | | Stores | the query | the rows | -| Freshness | always current | as of the last refresh | -| Read cost | the full query, every read | a table scan | +| Freshness | follows source-table reads | as of the last refresh | +| Read cost | executes the saved query | queries the stored result | | Write cost | none | a refresh | -Views are created in SQL, over the [Flight SQL endpoint](/sql#connecting) -- -there is no SDK method, because the thing a view stores *is* a SQL statement. -See [View](/sql/ddl/view) for `CREATE VIEW`, `DROP VIEW`, and listing. +Views are created in LanceDB Enterprise, either in SQL over the +[Flight SQL endpoint](/sql#connecting) or with the SDKs' `create_view` / +`createView`, which send the query to the deployment. Either way, the deployment +must have views enabled. See [View](/sql/ddl/view) for `CREATE VIEW`, +`DROP VIEW`, and listing. diff --git a/docs/web/quickstart.mdx b/docs/web/quickstart.mdx index 36001c97e..788dd6585 100644 --- a/docs/web/quickstart.mdx +++ b/docs/web/quickstart.mdx @@ -1,96 +1,55 @@ --- title: Quickstart sidebarTitle: "Quickstart" -description: "Get started with LanceDB in minutes." +description: "Install LanceDB, create a table, and run your first vector search." icon: rocket --- -import { - PyConnect, - PyConnectAsync, - PyConnectEnterpriseQuickstart, - PyConnectObjectStorage, - PyConnectObjectStorageAsync, - RsConnect, - RsConnectEnterpriseQuickstart, - RsConnectObjectStorage, - TsConnect, - TsConnectEnterpriseQuickstart, - TsConnectObjectStorage, -} from '/snippets/connection.mdx'; -import { - PyQuickstartData, - PyQuickstartComputedFeature, - PyQuickstartTraining, - PyQuickstartTrainingProjection, - PyQuickstartCreateTable, - PyQuickstartCreateTableAsync, - PyQuickstartAddFeature, - PyQuickstartCurateWithMetadata, - PyQuickstartMultimodalBytes, - PyQuickstartQueryFeature, - PyQuickstartVectorSearch1, - PyQuickstartVectorSearch1Async, - PyQuickstartOutputPandas, - RsQuickstartAddFeature, - RsQuickstartCurateWithMetadata, - RsQuickstartCreateTable, - RsQuickstartData, - RsQuickstartDefineStruct, - RsQuickstartMultimodalBytes, - RsQuickstartQueryFeature, - RsQuickstartVectorSearch1, - TsQuickstartAddFeature, - TsQuickstartCurateWithMetadata, - TsQuickstartCreateTable, - TsQuickstartData, - TsQuickstartMultimodalBytes, - TsQuickstartQueryFeature, - TsQuickstartVectorSearch1, -} from '/snippets/quickstart.mdx'; +import { PyConnect, TsConnect, RsConnect, PyConnectEnterprisePlaceholders, TsConnectEnterprisePlaceholders, RsConnectEnterprisePlaceholders } from '/snippets/connection.mdx'; +import { PyQuickstartData, PyQuickstartCreateTable, PyQuickstartAddData, PyQuickstartVectorSearch1, TsQuickstartData, TsQuickstartCreateTable, TsQuickstartAddData, TsQuickstartVectorSearch1, RsQuickstartDefineStruct, RsQuickstartData, RsQuickstartCreateTable, RsQuickstartAddData, RsQuickstartVectorSearch1 } from '/snippets/quickstart.mdx'; -As described in [the landing page](/), LanceDB provides one data layer for -curation, feature engineering, search and retrieval, and model training. Whether you are preparing -training data, building a RAG or agentic retrieval system, reviewing examples, or adding model-generated -features, you'll work with the same underlying table and search primitives. +This quickstart uses a local database and works without an account or API key. -Let's get started in just a few steps! +Follow the steps below, or jump to the [complete examples](#code) to copy a runnable program. ## 1\. Install LanceDB {#1-install-lancedb} -Install LanceDB in your client SDK. +Choose your language. Python uses Polars to display the search results. TypeScript needs Node.js 22.18 or later to run the `.mts` example directly. -```bash pip icon="terminal" -pip install lancedb +```bash Python (pip) icon="python" +pip install lancedb polars ``` -```bash uv icon="terminal" -uv add lancedb - -# Or, in an existing virtual environment: -uv pip install lancedb +```bash Python (uv) icon="python" +uv venv +uv pip install lancedb polars ``` -```bash TypeScript icon=js +```bash TypeScript icon="square-js" npm install @lancedb/lancedb ``` -```bash Rust icon=Rust -cargo add lancedb +```bash Rust icon="rust" +cargo init --name lancedb_quickstart +cargo add lancedb@0.31.0 --features polars,remote +cargo add tokio --features macros,rt-multi-thread ``` +Run the Rust commands in a new, empty directory. The `polars` feature provides DataFrame output; `remote` also lets this example connect to Enterprise. + + ### Python pre-release builds {#python-pre-release-builds} To pick up the latest features and bug fixes before the next stable release, install a pre-release from LanceDB's Fury index. -```bash pip icon="terminal" +```bash Python (pip) icon="terminal" pip install --pre --extra-index-url https://pypi.fury.io/lancedb/ lancedb ``` -```bash uv icon="terminal" +```bash Python (uv) icon="terminal" uv venv uv pip install --prerelease allow --index https://pypi.fury.io/lancedb/ lancedb @@ -105,436 +64,237 @@ for more than 6 months after release. For real-world workloads, we recommend you as far as possible. - +### Older x86_64 CPUs {#older-x86-cpus} The default `lancedb` wheel targets `x86-64-haswell` (AVX2 + FMA + F16C). On older x86_64 CPUs without AVX2 (Intel Sandy Bridge, Ivy Bridge, Westmere; AMD Bulldozer, Piledriver, Steamroller), `import lancedb` crashes with `Illegal instruction`. Install `lancedb-compat` instead. It exposes the same API (`import lancedb` still works), is built at the -`x86-64-v2` baseline, and uses runtime SIMD dispatch to still leverage AVX2, FMA, or AVX-512 when available: +`x86-64-v2` baseline, and uses runtime SIMD dispatch to still use AVX2, FMA, or AVX-512 when available: -```bash pip icon="terminal" +```bash Python (pip) icon="terminal" pip install lancedb-compat ``` The two packages share the same `lancedb/` namespace and conflict at install time, so pick one. To switch, uninstall the other first (`pip uninstall lancedb && pip install lancedb-compat`). + + -## 2\. Connect to a LanceDB database {#2-connect-to-a-lancedb-database} +## 2\. Connect to a database {#2-connect-to-a-lancedb-database} -LanceDB supports several URI patterns to connect to a database. +### Use a local directory {#connect-via-local-directory-path} -- A local filesystem path (when using it as an embedded library) -- A `db://...` URI (when using LanceDB Enterprise) -- An object storage URI: `s3://...`, `gs://...`, or `az://...` (when connecting directly from the client SDK) +LanceDB creates an `ex_lancedb` directory to store your tables. No server is needed. -### Connect via local directory path {#connect-via-local-directory-path} - -The simplest way to begin is to use LanceDB as an embedded library. Import LanceDB in your -client SDK of choice and point to a local directory path. - - - + + {PyConnect} - - - {PyConnectAsync} - - + {"import * as lancedb from \"@lancedb/lancedb\";\n\n"} {TsConnect} - - { "use lancedb::connect;\n" } - { "\n" } + {"use lancedb::connect;\n\n"} {RsConnect} - + +The [complete examples](#code) include the imports and entry points for each language. For Python's async API, see [Connecting to a database](/basics/connect). -### Connect via object storage URIs {#connect-via-object-storage-uris} - -You can also connect directly to object storage from the client SDK: - - - - {PyConnectObjectStorage} - - - - {PyConnectObjectStorageAsync} - - - - {TsConnectObjectStorage} - - - - {RsConnectObjectStorage} - - - -For credentials, endpoints, and provider-specific options, see -[Configuring storage](/deployment/storage). - + ### Connect to LanceDB Enterprise {#connect-to-lancedb-enterprise} -If you're using LanceDB Enterprise, you can connect to the remote database using the -`db://` URI along with the API key, region, and cluster endpoint you received from the -LanceDB team. Pass the cluster endpoint via `host_override` so the client routes -requests to your deployment. +Use the database URI, API key, and endpoint provided by your administrator or the LanceDB team. Replace the placeholders below with those details. Use the deployment's region if provided; otherwise keep `us-east-1`. - + - { "import lancedb\n\n" } - {PyConnectEnterpriseQuickstart} + {PyConnectEnterprisePlaceholders} - - { "import * as lancedb from \"@lancedb/lancedb\";\n\n" } - {TsConnectEnterpriseQuickstart} + {"import * as lancedb from \"@lancedb/lancedb\";\n\n"} + {TsConnectEnterprisePlaceholders} - - { "use lancedb::connect;\n\n" } - {RsConnectEnterpriseQuickstart} + {"use lancedb::connect;\n\n"} + {RsConnectEnterprisePlaceholders} - + -`host_override` is the full URL of your cluster endpoint, including the scheme -(`https://`) and a port if your deployment listens on a non-default one -(e.g. `https://your-enterprise-endpoint.com:443`). If you don't have the -endpoint, [contact the LanceDB team](https://www.lancedb.com/contact). +In application code, load credentials from environment variables or a secrets manager. See [connection settings](/basics/connect#enterprise) for an example. -To learn more about `RemoteTable` semantics and how Enterprise differs operationally from -embedded LanceDB, see the [Enterprise overview](/enterprise). - -## 3\. Create a new table {#3-create-a-new-table} - -Let's create a small table of characters from the kingdom of Camelot. Each row stores source text, -metadata, structured fields, and a vector embedding in the same LanceDB table. - - -The embeddings we use in this example are synthetic and for demonstration purposes only. In a real AI -data workflow, you would generate them from text, images, audio, or video using an embedding model of choice. - - -Each row has source text, metadata, structured fields, and a vector: - -```json -{ - "id": "2", - "name": "Merlin", - "role": "Wizard", - "description": "Advisor and prophet with deep magical knowledge.", - "stats": { "strength": 2, "magic": 5, "leadership": 4, "wisdom": 5 }, - "vector": [0.2, 0.9, 0.4, 0.9] -} -``` - -The full raw records are included below: - - - - - {PyQuickstartData} - - - - {TsQuickstartData} - - - - {RsQuickstartDefineStruct} - {RsQuickstartData} - - +Need a deployment? [Contact the LanceDB team](https://www.lancedb.com/contact). For differences between local and server execution, see [Moving to LanceDB Enterprise](/basics/sdk#moving-to-enterprise). -You can now create a LanceDB table from those records. The code below creates a LanceDB table -with the appropriate schema and ingests the data. + +### Connect via object storage URIs {#connect-via-object-storage-uris} - - +Use an `s3://`, `gs://`, or `az://` URI in place of the local path. See [Configuring storage](/deployment/storage) for credentials and provider-specific options. + + +## 3\. Create a table and add data {#3-create-a-new-table} + +Each row describes a character and stores a vector alongside their name, description, and stats. + + +These vectors are made up for this example. In your application, use an embedding model to generate vectors from your data. + + + + + + {PyQuickstartData} + + + {TsQuickstartData} + + + {RsQuickstartDefineStruct} + {RsQuickstartData} + + + + +Create a `characters` table from those records: + + + {PyQuickstartCreateTable} - - - {PyQuickstartCreateTableAsync} - - {TsQuickstartCreateTable} - {RsQuickstartCreateTable} - + -## 4\. Semantic search {#4-semantic-search} +This example uses overwrite mode so you can run it again. It replaces any existing `characters` table in the database you connect to. -Search is a useful capability for all kinds of AI data pipelines. Below, we do a vector similarity -search for samples similar to a "_wise magical advisor_" (transforming the natural language query to -an embedding), and project only the columns needed by the next step. +Add one more character: -Search (which requires random access) is a ubiquitous access pattern that appears in many workloads: -whether you're building a RAG or recommendation system, serving agent memory, or curating a training -dataset. + + + {PyQuickstartAddData} + + + {TsQuickstartAddData} + + + {RsQuickstartAddData} + + - - +## 4\. Search the table {#4-semantic-search} + +Find the two vectors closest to the query vector. This example uses a fixed vector; it does not call an embedding model. + + + {PyQuickstartVectorSearch1} - - - {PyQuickstartVectorSearch1Async} - - {TsQuickstartVectorSearch1} - {RsQuickstartVectorSearch1} - + -The example for Python above shows how to convert results to a Polars DataFrame. -Depending on your language, you can collect query results as a list/array of objects or DataFrames -to be used downstream in your application. +Merlin and Morgana come back first. Python and Rust display a Polars DataFrame; TypeScript displays an array of results. The `_distance` column measures how far each result is from your query vector, with smaller values indicating a closer match. - - Use the `to_pandas()` method to convert query results into a Pandas DataFrame. - - {PyQuickstartOutputPandas} - +## Complete examples {#code} + +Each example below creates the local database, adds four characters, and prints the two closest matches. Use the dependencies from step 1. + + +Save this as `quickstart.py` and run `python quickstart.py`. If you installed with uv, run `uv run quickstart.py`. + + + + {PyConnect} + {PyQuickstartData} + {PyQuickstartCreateTable} + {PyQuickstartAddData} + {PyQuickstartVectorSearch1} + + -## 5\. Curation {#5-curation} + +Save this as `quickstart.mts` and run `node quickstart.mts` from the directory where you installed LanceDB. -Searching for relevant results can be more useful when combined with metadata filters. -In this tiny example, we filter to examples with high `magic` stats. - - - - {PyQuickstartCurateWithMetadata} + + + {"import * as lancedb from \"@lancedb/lancedb\";\n\n"} + {TsConnect} + {TsQuickstartData} + {TsQuickstartCreateTable} + {TsQuickstartAddData} + {TsQuickstartVectorSearch1} + + - - {TsQuickstartCurateWithMetadata} + +Replace `src/main.rs` with this program, then run `cargo run` from your project directory. The schema and Arrow conversion helpers are included. + + + + {"use lancedb::connect;\n\n"} + {RsQuickstartDefineStruct} + {"\n#[tokio::main]\nasync fn main() -> Result<(), Box> {\n"} + {RsConnect} + {RsQuickstartData} + {RsQuickstartCreateTable} + {RsQuickstartAddData} + {RsQuickstartVectorSearch1} + {"\nOk(())\n}\n"} - - - {RsQuickstartCurateWithMetadata} - - - -When working with large datasets, it's common to use the same pattern to filter on quality labels, -train/eval splits, numeric fields, categorical values, timestamp windows, or generated tags and labels. - -## 6\. Add a derived feature {#6-add-a-derived-feature} - -Feature engineering is the process of cleaning up your data and creating new signals that -help your model learn, make better predictions, or your agent retrieve more useful information. -In the example below, we add a `power_score` column from the structured `stats` fields. -Lance supports data evolution, so you can add new columns without rewriting the entire table. - - - - {PyQuickstartAddFeature} - - - - {TsQuickstartAddFeature} - - - - { "use lancedb::table::NewColumnTransform;\n\n" } - {RsQuickstartAddFeature} - - - -Next, you can query a compact view of the new feature: - - - - {PyQuickstartQueryFeature} - - - - {TsQuickstartQueryFeature} - - - - {RsQuickstartQueryFeature} - - - -| name | role | power_score | -| --- | --- | --- | -| King Arthur | King | 3.5 | -| Merlin | Wizard | 4.0 | -| Sir Lancelot | Knight | 3.0 | - -The same workflow is used for data preparation tasks when adding derived features, cached model signals, review scores, or dataset -quality indicators. - -### Declare now, compute later {#computed-columns} - -The call above computes every row as part of the call, which is fine for three -rows. On a table large enough that you would rather choose *when* the work -happens, declare a column instead and fill it separately. The stored `power_score` -column stays available; declare a separate -`combat_score` column from magic and strength, then refresh it: - - -{PyQuickstartComputedFeature} - - -A refresh gives values to rows that have none and recomputes only rows whose -inputs changed, so appending more characters and refreshing again fills just -the new ones -- it does not recompute the rows that are already current. That -is what makes a backfill safe to retry and cheap to repeat as data arrives. - -On a deployment the same refresh runs server-side, on a cluster, as a -[job](/basics/jobs) you can walk away from: - -```python -job = table.refresh_column_async("combat_score") -job.wait() -``` - -See [Computed columns](/feature-engineering/add-computed-column) for expressions, -and [Functions](/management/function) for filling a column with Python --- an embedding model, a captioner, anything you can register. - -## 7\. Store multimodal data {#7-store-multimodal-data} - -Multimodal data is a first-class citizen in LanceDB. Binary data (image, audio, video, etc.) is -stored as blobs or inline Arrow binary types in a LanceDB column, and they benefit from the same -table operations and data versioning semantics as other data types. All the data is governed -in the same table, so you can search, filter, and retrieve multimodal records together with structured -fields, metadata, and embeddings. - -In this example, the -[`lancedb/magical_kingdom`](https://huggingface.co/datasets/lancedb/magical_kingdom) dataset stores -character images, descriptions, structured stats, image embeddings, and text embeddings together. - -Say we downloaded the image for Sir Lancelot from that dataset locally. You can read the image bytes -in your client SDK and store them in a LanceDB column. The image bytes can be used for downstream tasks -like retrieval, evaluation, or training. - -
- Sir Lancelot from the lancedb/magical_kingdom dataset -
- -These snippets load the local image file and store the bytes in an `image` column: - - - - {PyQuickstartMultimodalBytes} - - - - {TsQuickstartMultimodalBytes} - - - - {RsQuickstartMultimodalBytes} - - - -For more examples, see [Data types](/management/table#multimodal). - -## 8\. Train on the result {#8-train} - -The point of the previous steps is this one. The table you searched, filtered, -and added features to is directly usable as a PyTorch -[Dataset](https://docs.pytorch.org/docs/stable/data.html#torch.utils.data.Dataset) --- there is no export step, and no second copy of the data to keep in sync. - -Install PyTorch with `pip install torch`. Select the numeric feature columns -before constructing the loader; the full table also contains strings, -structured stats, and vectors. - - -{PyQuickstartTraining} - - -```python -for batch in dataloader: - ... # train with batch["power_score"] and batch["combat_score"] -``` - -Because the loader reads from the table, the feature columns you just -materialized are read straight off storage rather than recomputed every epoch. -That is the whole argument for doing feature engineering in the lakehouse: an -expensive transform runs once, is versioned with the data, and every subsequent -epoch is a read. - -Projection matters here. If a training step only needs `power_score`, select -that column so the descriptions, stats, and vectors are excluded: - - -{PyQuickstartTrainingProjection} - - -See [Training](/training/data-loading) for shuffling, multi-worker loading, and reading from -a remote table. - -## Code {#code} - -See the full code for these examples (including helper functions) in the -`quickstart` file for the appropriate client language in the -[files provided in the repo](https://github.com/lancedb/docs/tree/main/tests). - + + ## Next steps {#next-steps} -You've gone from a table, through search and curation, to derived features and a -training loop -- in one table, with no export step between any two stages. That -arc is the argument for a multimodal lakehouse, and every stage of it scales -past this toy example without changing shape. -Continue to the guides to build on it with schema options, versioning, indexing, -full-text and hybrid search, reranking, and server-side feature engineering. +You've created a table, added rows, and searched it. Continue with the guides below. + +### Filter results {#5-curation} + +Combine vector search with metadata filters, such as `stats.magic >= 4`. See [SQL filtering](/search/filtering). + +### Extend your table {#extensions} + +#### Add a derived feature {#6-add-a-derived-feature} + +Add columns calculated from existing fields. See [Adding columns](/management/table#schema-add-new-columns). + +#### Computed columns {#computed-columns} + +Declare a column and choose when to fill or refresh it. See [Computed columns](/feature-engineering/add-computed-column). + +#### Store multimodal data {#7-store-multimodal-data} + +Store images, audio, and other binary data alongside vectors and metadata. See [Multimodal data types](/management/table#multimodal). + +#### Train on your data {#8-train} + +Read table data into PyTorch for training. See [Data loading](/training/data-loading). - - Build on this quickstart with table creation, updates, and schema tips. + +Create, update, and manage your tables. - - Learn how to build Retrieval-Augmented Generation (RAG) applications using LanceDB. + +Use retrieved context in a language model application. - - Create vector, full-text, and scalar indexes to speed up queries on larger datasets. + +Speed up queries on larger datasets. - - Use LanceDB for projected, shuffled, random-access reads in training workflows. + +Generate vectors from your own data. diff --git a/docs/web/reranking/index.mdx b/docs/web/reranking/index.mdx index e17b43552..26db30442 100644 --- a/docs/web/reranking/index.mdx +++ b/docs/web/reranking/index.mdx @@ -1,96 +1,100 @@ --- title: "Reranking Search Results" sidebarTitle: "Reranking" -description: "Use a reranker to improve search relevance." +description: "Choose a reranker, apply it to search candidates and evaluate relevance and latency." icon: "sort-amount-down" keywords: ["re-ranking", "reranker", "rerank"] +availability: + oss: available + enterprise: available + summary: >- + Reranking is available with OSS and Enterprise. --- import { PyRerankingLinearCombinationUsage, PyRerankingRrfUsage, - PyRerankingCohereUsage, PyRerankingMrrUsage, PyRerankingCrossEncoderUsage, } from '/snippets/integrations.mdx'; +import { PyRerankingCohereUsage } from '/snippets/provider_examples.mdx'; -Reranking is the process of re-ordering search results to improve relevance, often using a -different model than the one used for the initial search. LanceDB has built-in support for reranking -with models from Cohere, Sentence-Transformers, and more. +import { PyHybridSearch, TsHybridSearchBody, TsHybridSearchTail } from '/snippets/search_examples.mdx'; +import { RsExampleBody as RsHybridSearchBody, RsExampleTail as RsHybridSearchTail } from '/snippets/search_hybrid_search.mdx'; -### Quickstart {#quickstart} +Reranking changes the order of records returned by an initial search. It can +combine several ranked lists or use a model to score each candidate against +the query. It cannot recover records that the initial searches did not retrieve. -To use a reranker, you run a search and pass the results to the `rerank()` method. The examples below -move from the simplest, model-free rerankers to a model-based one. Each is a complete, runnable script. +## Choose a reranker -**1. Linear combination (simplest).** `LinearCombinationReranker` normalizes the vector and full-text -scores and blends them with a single `weight` (default `0.7`, favoring the vector score). It runs no -model, and reranks [hybrid search](/search/hybrid-search) results. +- **Rank fusion:** start with RRF for hybrid search. It combines vector and + full-text rankings without a model or a provider API call. +- **Model-based reranking:** try a cross encoder or hosted reranking model + when you need a different relevance judgment over the retrieved text. + Compare the relevance gain with added latency and provider cost. +- **Custom ranking:** implement the reranker interface when your application + needs its own scoring policy. + + + + + +These examples use three sample records and hand-written vectors, so no +embedding service is needed. Each program creates a temporary local table +and removes it when finished. + +- **Python:** Install with `pip install lancedb`, save as `rerank_search.py`, then run `python rerank_search.py`. +- **TypeScript:** Install with `npm install @lancedb/lancedb`, save as `rerank_search.mts`, then run `node rerank_search.mts` with Node.js 22.18 or later. +- **Rust:** Download and extract the [complete Rust examples](/static/code/search-rust.zip), then run `cargo run --locked --bin hybrid_search` from the extracted directory. - - {PyRerankingLinearCombinationUsage} - + +{PyHybridSearch} + + +{TsHybridSearchBody + "\n" + TsHybridSearchTail} + + +{RsHybridSearchBody + "\n" + RsHybridSearchTail} + -**2. Reciprocal Rank Fusion.** `RRFReranker` fuses results by rank position instead of raw score, so it -sidesteps having to make vector and full-text scores comparable. It loads no model either, and it's the -default reranker for hybrid search. +The result includes `_relevance_score` in descending order. RRF is already the +hybrid-search default; passing it explicitly shows where to substitute another +reranker. These scores express a ranking, not a probability of correctness. - - - {PyRerankingRrfUsage} - - + -**3. Cohere (model-based).** For higher relevance, a model-based reranker scores each result against the -query with a trained model. `CohereReranker` works with vector, full-text, or hybrid search, and needs -the `cohere` package plus either `COHERE_API_KEY` in the environment or an `api_key` argument. +## Supported rerankers {#supported-rerankers} - - - {PyRerankingCohereUsage} - - +Use rank fusion to combine search results without a model, or a model-based +reranker to score each document against the query. Reranking runs in your +application after retrieval, including when the table is in Enterprise. -Reach for the model-free rerankers (`LinearCombinationReranker`, `RRFReranker`) when cost and latency -matter most; reach for a model-based one like `CohereReranker` or `CrossEncoderReranker` when you need -higher relevance and can afford to score every query and document pair with a model. +| Reranker | Method | SDK | +|:---------|:-------|:----| +| `RRFReranker` | Reciprocal rank fusion | Python, TypeScript, Rust | +| `MRRReranker` | Weighted reciprocal rank fusion | Python | +| `LinearCombinationReranker` (deprecated) | Blend normalized scores | Python | +| `CrossEncoderReranker` | Score query-document pairs with a local cross encoder | Python | +| `ColbertReranker`, `AnswerdotaiRerankers` | Local model reranking | Python | +| Provider rerankers | Call a hosted model, such as Cohere, Jina or Voyage AI | Python | -### Supported Rerankers {#supported-rerankers} - -LanceDB supports the following rerankers out of the box. The first three are score-based and run no -model; the rest are model-based. The built-in rerankers are documented in this section; the hosted -providers that need an API key live under [integrations](/integrations/reranking). - -| Reranker | Default model | -| --------------------------- | -------------------------------------- | -| `RRFReranker` | None (reciprocal rank fusion) | -| `LinearCombinationReranker` | None (weighted score blend) | -| `MRRReranker` | None (weighted reciprocal rank) | -| `CohereReranker` | `rerank-english-v3.0` | -| `CrossEncoderReranker` | `cross-encoder/ms-marco-TinyBERT-L-6` | -| `ColbertReranker` | `colbert-ir/colbertv2.0` | -| `AnswerdotaiRerankers` | `answerdotai/answerai-colbert-small-v1`| -| `JinaReranker` | `jina-reranker-v2-base-multilingual` | -| `OpenaiReranker` | `gpt-4-turbo-preview` | -| `VoyageAIReranker` | No default (model name required) | -| `WatsonxReranker` | `cross-encoder/ms-marco-minilm-l-12-v2`| - -The model-based rerankers need their provider package installed, and the hosted ones -(`CohereReranker`, `JinaReranker`, `OpenaiReranker`, `VoyageAIReranker`, `WatsonxReranker`) also need an API key, passed -as an `api_key` argument or set in the provider-specific environment variable. +Local model rerankers need their model runtime packages. Hosted rerankers need +provider credentials and make API calls. See [reranking integrations](/integrations/reranking) +for provider setup and model selection; specify a model supported by your provider. Rerankers add `_relevance_score` and return rows ordered by descending relevance. Python rerankers -accept `return_score="relevance"` or `return_score="all"`; use `"all"` when you want to keep the +accept a `return_score` option; use `"all"` where supported to keep the original vector distance or FTS score columns for debugging. Model-based rerankers read from `column="text"` by default, so either return that column in the search results or pass a different column. -Use `refine_factor` on vector or hybrid queries when you're reranking approximate IVF-PQ results and -want a larger candidate pool before the final ranking step. A value of `3` asks LanceDB to fetch -`limit * 3` candidates, refine them with the full vectors, and keep the requested `limit`. Higher -values can improve recall, but they also increase query latency. +`refine_factor` expands the vector index's internal candidate pool and rescores +those candidates using full vectors before keeping the requested limit. It +can improve the candidates available for reranking, but it does not multiply +the final result count. Compare retrieval recall and total query latency. **SDK coverage differs across languages** @@ -106,7 +110,9 @@ yourself. ### Multi-vector reranking {#multi-vector-reranking} -Most rerankers support reranking based on multiple vectors. To rerank based on multiple vectors, you can pass a list of vectors to the `rerank` method. Here's an example of how to rerank based on multiple vector columns using the `CrossEncoderReranker`: +Use `rerank_multivector()` to combine result sets from several vector queries. +The example below assumes `table` has three vector columns with embedding +metadata and uses a cross encoder to rerank their candidates. ```python Python icon="python" @@ -132,15 +138,24 @@ input result sets is missing the `_rowid` column. Therefore, it's recommended to - `RRFReranker.rerank_multivector(...)` always requires `_rowid` on its inputs, regardless of the `deduplicate` flag. -## Creating Custom Rerankers {#creating-custom-rerankers} +## Implement a custom ranking policy {#creating-custom-rerankers} -LanceDB also allows you to create custom rerankers by extending the base `Reranker` class. The custom reranker -should implement the `rerank` method that takes a list of search results and returns a reranked list of -search results. This is covered in more detail in the [creating custom rerankers](/reranking) section. +Python custom rerankers implement `rerank_hybrid()` and can optionally implement +`rerank_vector()` and `rerank_fts()`. TypeScript and Rust expose hybrid reranking +interfaces. See [the interface requirements](#custom) before implementing one. + +## Algorithm options + +The following sections describe the Python constructor options and examples. +Provider and local-model examples require their additional packages and model +or credential setup. They are separate programs, not steps to concatenate. ## Reciprocal rank fusion {#rrf} -## Reciprocal Rank Fusion Reranker + + + + **Reciprocal Rank Fusion (RRF)** is a model-free way to merge several ranked result lists into a single ordering. Rather than comparing raw similarity scores (which aren't directly comparable @@ -162,10 +177,10 @@ why it's the default reranker for LanceDB hybrid search. The implementation foll -### Accepted Arguments {#rrf-accepted-arguments} +### Python arguments {#rrf-accepted-arguments} | Argument | Type | Default | Description | | --- | --- | --- | --- | -| `K` | `int` | `60` | A constant used in the RRF formula (default is 60). Experiments indicate that k = 60 was near-optimal, but that the choice is not critical. | +| `K` | `int` | `60` | A constant used in the RRF formula (default is 60). Larger values reduce the difference in contribution between nearby rank positions. | | `return_score` | `str` | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score`. If "all", will return all scores from the vector and FTS search along with the relevance score. | `K` must be greater than `0`. In TypeScript, construct the built-in reranker with @@ -185,18 +200,24 @@ You can specify the type of scores you want the reranker to return. The followin #### Hybrid Search {#rrf-hybrid-search} |`return_score`| Status | Description | | --- | --- | --- | -| `relevance` | ✅ Supported | Returned rows only have the `_relevance_score` column. | -| `all` | ✅ Supported | Returned rows have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). | +| `relevance` | ✅ Supported | Keeps `_relevance_score` as the score column alongside the selected data columns. | +| `all` | ✅ Supported | Returned rows have vector(`_distance`) and FTS(`_score`) along with Hybrid Search score(`_relevance_score`). | + + + ## Linear combination {#linear-combination} -## Linear Combination Reranker + -> **Note:** This reranker is deprecated. Use the `RRFReranker` if you need a score-based reranker. + + + +> **Note:** This reranker is deprecated. Use `RRFReranker` to combine vector and full-text rankings. The Linear Combination Reranker combines the results of semantic and full-text search using a linear combination of the scores. The weights for the linear combination can be specified, and defaults to 0.7, i.e, 70% weight for semantic search and 30% weight for full-text search. -> **Note:** Supported query type – Hybrid search only. +> **Note:** Supported query type: Hybrid search only. @@ -205,7 +226,7 @@ The Linear Combination Reranker combines the results of semantic and full-text s -### Accepted Arguments {#linear-combination-accepted-arguments} +### Python arguments {#linear-combination-accepted-arguments} | Argument | Type | Default | Description | | --- | --- | --- | --- | | `weight` | `float` | `0.7` | The weight to use for the semantic search score. The weight for the full-text search score is `1 - weight`. | @@ -222,16 +243,22 @@ You can specify the type of scores you want the reranker to return. The followin #### Hybrid Search {#linear-combination-hybrid-search} |`return_score`| Status | Description | | --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column | -| `all` | ✅ Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_distance`) | +| `relevance` | ✅ Supported | Keeps `_relevance_score` alongside the selected data columns | +| `all` | ✅ Supported | Results have vector(`_distance`) and FTS(`_score`) along with Hybrid Search score(`_relevance_score`) | + + + ## MRR {#mrr} -## MRR Reranker + + + + This reranker uses the Mean Reciprocal Rank (MRR) algorithm to combine and rerank search results from vector and full-text search. You can use this reranker by passing `MRRReranker()` to the `rerank()` method. The MRR algorithm calculates the average of reciprocal ranks across different search results, providing a balanced way to merge results from multiple ranking systems. -> **Note:** Supported query types – Hybrid and Multivector search. +> **Note:** Supported query types: Hybrid and Multivector search. @@ -239,7 +266,7 @@ This reranker uses the Mean Reciprocal Rank (MRR) algorithm to combine and reran -### Accepted Arguments {#mrr-accepted-arguments} +### Python arguments {#mrr-accepted-arguments} | Argument | Type | Default | Description | | --- | --- | --- | --- | | `weight_vector` | `float` | `0.5` | Weight for vector search results (0.0 to 1.0). | @@ -258,21 +285,27 @@ You can specify the type of scores you want the reranker to return. The followin #### Hybrid Search {#mrr-hybrid-search} |`return_score`| Status | Description | | --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | -| `all` | ✅ Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). | +| `relevance` | ✅ Supported | Keeps `_relevance_score` alongside the selected data columns. | +| `all` | ✅ Supported | Results have vector(`_distance`) and FTS(`_score`) along with Hybrid Search score(`_relevance_score`). | #### Multivector Search {#mrr-multivector-search} |`return_score`| Status | Description | | --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | +| `relevance` | ✅ Supported | Keeps `_relevance_score` alongside the selected data columns. | | `all` | ✅ Supported | Results have vector distances from all searches along with `_relevance_score`. | + + + ## Cross encoder {#cross-encoder} -## Cross Encoder Reranker + + + + This reranker uses Cross Encoder models from sentence-transformers to rerank the search results. You can use this reranker by passing `CrossEncoderReranker()` to the `rerank()` method. -> **Note:** Supported query types – Hybrid, Vector, and FTS. +> **Note:** Supported query types: Hybrid, Vector, and FTS. @@ -281,7 +314,7 @@ This reranker uses Cross Encoder models from sentence-transformers to rerank the -### Accepted Arguments {#cross-encoder-accepted-arguments} +### Python arguments {#cross-encoder-accepted-arguments} | Argument | Type | Default | Description | | --- | --- | --- | --- | | `model_name` | `str` | `"cross-encoder/ms-marco-TinyBERT-L-6"` | The name of the reranker model to use.| @@ -299,20 +332,23 @@ You can specify the type of scores you want the reranker to return. The followin #### Hybrid Search {#cross-encoder-hybrid-search} |`return_score`| Status | Description | | --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | -| `all` | ❌ Not Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). | +| `relevance` | ✅ Supported | Keeps `_relevance_score` alongside the selected data columns. | +| `all` | ✅ Supported | Keeps `_distance`, `_score` and `_relevance_score` alongside the data columns. | #### Vector Search {#cross-encoder-vector-search} |`return_score`| Status | Description | | --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | +| `relevance` | ✅ Supported | Keeps `_relevance_score` alongside the selected data columns. | | `all` | ✅ Supported | Results have vector(`_distance`) along with Hybrid Search score(`_relevance_score`). | #### FTS Search {#cross-encoder-fts-search} |`return_score`| Status | Description | | --- | --- | --- | -| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. | -| `all` | ✅ Supported | Results have FTS(`score`) along with Hybrid Search score(`_relevance_score`). | +| `relevance` | ✅ Supported | Keeps `_relevance_score` alongside the selected data columns. | +| `all` | ✅ Supported | Results have FTS(`_score`) along with Hybrid Search score(`_relevance_score`). | + + + ## Building a custom reranker {#custom} @@ -321,11 +357,9 @@ minimum, you need to implement `rerank_hybrid()`, which is the logic that combin full-text search results. Beyond that, you can optionally implement `rerank_vector()` and `rerank_fts()` if you want your reranker to also handle pure vector or pure full-text searches. -Decide up front which surfaces — hybrid, pure vector, or pure full-text — your reranker should -cover, and only override the ones you need. The base class leaves `rerank_vector()` and -`rerank_fts()` unimplemented, so calling `.rerank(...)` on a single-modality search you haven't -overridden raises `NotImplementedError` rather than silently returning unsorted results. That's a -useful guard, but worth knowing about before you wire up a query path you didn't plan for. +Implement the hooks your query paths call. The base class raises +`NotImplementedError` for vector-only or FTS-only reranking unless you +implement those methods. The Python base class exposes hybrid, vector-only, and FTS-only rerank hooks. TypeScript and Rust currently expose the custom reranker interface for hybrid reranking. In Rust, a custom reranker must @@ -350,55 +384,28 @@ For vector-only reranking in Python, pass a text query to `.rerank(..., query_st The vector query itself may be numeric, but `rerank_vector(query, vector_results)` still receives a string query so your reranker can score each candidate against the user's text intent. -Below, we show the pseudocode of a custom reranker that combines the results of semantic and full-text -search using a linear combination of the scores: +The method contract is: - -```python Python icon="python" -from lancedb.rerankers import Reranker -import pyarrow as pa +| Method | Input | Required output | +|:-------|:------|:----------------| +| `rerank_hybrid` | Query text, vector results and FTS results | A merged Arrow table ordered by descending `_relevance_score` | +| `rerank_vector` | Query text and vector results | An Arrow table ordered by descending `_relevance_score` | +| `rerank_fts` | Query text and FTS results | An Arrow table ordered by descending `_relevance_score` | -class MyReranker(Reranker): - def __init__(self, param1, param2, ..., return_score="relevance"): - super().__init__(return_score) - self.param1 = param1 - self.param2 = param2 +Implement only the single-search hooks your application uses. The base class +raises `NotImplementedError` for those hooks until they are implemented. - def rerank_hybrid(self, query: str, vector_results: pa.Table, fts_results: pa.Table): - # Use the built-in merging function - combined_result = self.merge_results(vector_results, fts_results) +### Extend an existing reranker {#custom-example} - # Do something with the combined results - # ... - - # Return the combined results - return combined_result - - def rerank_vector(self, query: str, vector_results: pa.Table): - # Do something with the vector results - # ... - - # Return the vector results - return vector_results - - def rerank_fts(self, query: str, fts_results: pa.Table): - # Do something with the FTS results - # ... - - # Return the FTS results - return fts_results -``` - - -### Example {#custom-example} - -As an example, let's build custom reranker that enhances the Cohere Reranker by accepting a filter -query, and accepts any other `CohereReranker` params as `kwargs`. +This class definition extends `CohereReranker` to exclude text patterns after +reranking. It requires the Cohere package and credentials when instantiated. +It can return fewer results because exclusions happen after retrieval; use a +query predicate instead when a metadata filter expresses the same restriction. ```python Python icon="python" from typing import List, Union -import pandas as pd +import pyarrow as pa from lancedb.rerankers import CohereReranker class ModifiedCohereReranker(CohereReranker): @@ -433,58 +440,42 @@ class ModifiedCohereReranker(CohereReranker): ``` - -Under the hood, `vector_results` and `fts_results` are PyArrow tables. You can learn more about -PyArrow tables [here](https://arrow.apache.org/docs/python). The advantage of PyArrow tables is their -interoperability -- you can easily convert them to Pandas/Polars DataFrames, `PyDict`, `PyList`, etc. +The reranker receives and returns PyArrow tables. Use `to_pandas()` for a +DataFrame transformation and `pa.Table.from_pandas()` to convert the result +back, preserving `_relevance_score` and the desired row order. -The benefits are also bidirectional -- just as you can easily convert PyArrow tables _to_ Pandas -DataFrames using the `to_pandas()` method -- you can perform DataFrame transformations -and just as easily convert the DataFrame back to PyArrow tables using `pa.Table.from_pandas()` method -as shown in the example above. - -## Evaluating hybrid search {#eval} +## Evaluate relevance, latency and cost {#eval} -Hybrid search is an often misused and/or misunderstood term. In this section, we're using -the definition of "hybrid search" to mean using a combination of keyword-based and vector search. -Because the vector search operates in a dense embedding space and keyword-based search operate -in a sparse embedding space, their relevance scores cannot be directly compared. -Combining results from multiple searches thus requires a reranking step. - -Before evaluating hybrid search, build an FTS index on the text column and use a table with embedding -metadata or explicit query vectors for the vector side. Otherwise the evaluation is measuring setup -fallbacks or errors rather than the reranker. +Compare rerankers on the same candidate inputs so you can distinguish ranking +changes from retrieval changes. For hybrid retrieval, prepare the text index +and query vectors before collecting those candidates. ### Reranking strategies {#eval-reranking-strategies} -There are two common approaches for reranking search results from multiple sources. +Compare these approaches: -- **Score-based**: Calculate final relevance scores from the individual search algorithm scores. Examples: Reciprocal Rank Fusion (the default in LanceDB), mean reciprocal rank fusion, and weighted linear combination of semantic and keyword-based search scores. +- **Rank fusion:** RRF and MRR combine rank positions from multiple result lists. +- **Score blending:** linear combination mixes normalized vector and text scores. +- **Model-based reranking:** a model scores each query-document pair. -- **Relevance-based**: Discards the existing scores and calculates the relevance of each search result-query pair. Example: Cross Encoder models - - -If you call `.rerank()` on a hybrid query without passing a reranker, LanceDB defaults to -`RRFReranker()` — a score-based reranker that uses Reciprocal Rank Fusion. This is the -score-based path most readers encounter first; `LinearCombinationReranker` is an alternative -score-based strategy you opt into explicitly. - +Hybrid search uses RRF by default. You do not need to call `rerank()` to select +it. Pass a reranker when you want a different algorithm or configuration. By default, rerankers return `_relevance_score`. Pass `return_score="all"` when a reranker supports it, and you also need the original vector or FTS scores for debugging. Evaluation code can rely on returned rows being ordered by descending `_relevance_score`. Empty reranked result sets still include the `_relevance_score` column. -The hybrid `rerank(...)` method also accepts a `normalize` argument that controls how the raw +The synchronous Python hybrid `rerank(...)` method also accepts a `normalize` argument that controls how the raw vector and FTS scores are made comparable before reranking: -- `normalize="score"` (the default) — normalizes the raw vector and FTS scores directly. -- `normalize="rank"` — converts each result list to ranks first, then normalizes. +- `normalize="score"` (the default); normalizes the raw vector and FTS scores directly. +- `normalize="rank"`; converts each result list to ranks first, then normalizes. -This choice materially affects score-based rerankers (such as `LinearCombinationReranker`), so -when you evaluate score-based strategies, treat `normalize` as a tunable hyperparameter -alongside the reranker itself. +Normalization matters for algorithms that use raw scores, such as linear +combination. RRF uses rank positions instead. Compare normalization settings +only when the chosen reranker uses those scores. For score-based evaluation, the main built-in knobs are: @@ -492,17 +483,18 @@ For score-based evaluation, the main built-in knobs are: - `MRRReranker(weight_vector=0.5, weight_fts=0.5)`: weighted reciprocal rank fusion where the weights must sum to `1.0`. - `LinearCombinationReranker(weight=0.7, fill=1.0)`: blends vector and FTS scores directly; `fill` controls how strongly to penalize a result missing from one side. -Even though there are many more strategies for reranking, there are no "universally best" -ones that work well for all cases, because reranking quality is dataset and application specific. -Evaluating whether a reranking strategy is a good fit is also a challenge. In the next -section, we discuss an example evaluation of different reranking strategies on a sample dataset. +A result on one dataset does not establish which reranker will work best on +another. Include representative queries and measure both retrieval quality +and ranking quality. -### Example evaluation {#eval-example-evaluation} +### Historical evaluation example {#eval-example-evaluation} -The table below shows our evaluation results from an experiment comparing multiple rerankers on +The tables below preserve an earlier experiment comparing multiple rerankers on ~800 hybrid search queries. This is a modified version of an evaluation script by [LlamaIndex](https://github.com/run-llama/finetune-embedding/blob/main/evaluate.ipynb) that measures -hit-rate \@ top-k. +hit-rate \@ top-k. The recorded results do not establish performance for +current models, SDK releases or your dataset; treat them as an example of an +evaluation format. #### Using OpenAI `text-embedding-ada-002` {#eval-using-openai-text-embedding-ada-002} @@ -531,14 +523,14 @@ Vector Search baseline: **0.59** -### Conclusion {#eval-conclusion} +### Compare results on your workload {#eval-conclusion} -The results show that the reranking methods can significantly improve the search relevance. However, -the improvement we saw was not consistent across all rerankers. In reality, the choice of reranker -likely depends on the dataset and the application. +Build a set of representative queries with known relevant records. Compare +retrieval alone with each reranker using the same candidate inputs and result +limit. Record a relevance metric such as recall or nDCG alongside latency and +provider cost. Include queries with no useful matches and queries containing +exact names or identifiers. -It's also important to note that the reranking methods are not a -replacement for the search methods they supplement. They are complementary and it's likely that you'd -have to tune them together to get the best results. The latency vs. recall tradeoff is also an -important factor to consider when choosing the reranker. Hopefully this evaluation -gives you a starting point for your own experiments with hybrid search in LanceDB! +If the relevant records never reach the reranker, adjust retrieval first. +Then change one ranking setting at a time and retain the configuration that +meets your application's quality and latency requirements. diff --git a/docs/web/search/explain.mdx b/docs/web/search/explain.mdx index b2fc12c04..383745e5b 100644 --- a/docs/web/search/explain.mdx +++ b/docs/web/search/explain.mdx @@ -1,40 +1,72 @@ --- title: "Explain a query plan" sidebarTitle: "Query Plans" -description: "Read the plan LanceDB resolved for a query, and measure it with analyze." +description: "Inspect query execution, identify expensive work and compare changes." icon: "map" +availability: + oss: available + enterprise: available + summary: >- + Query plans are available in OSS and Enterprise. --- -`explain_plan` prints the plan LanceDB resolved for a query without executing -it. It answers structural questions -- did the filter push into the scan, is the -vector index being used, which columns are actually read -- and costs nothing to -run, so it is the first thing to reach for when a query behaves unexpectedly. +import { PyQueryPlan, TsQueryPlanBody, TsQueryPlanTail } from '/snippets/search_examples.mdx'; +import { RsExampleBody as RsQueryPlanBody, RsExampleTail as RsQueryPlanTail } from '/snippets/search_query_plan.mdx'; -To measure a query rather than read it, use -[`analyze_plan`](/search/explain), which executes it and annotates every -operator with runtime metrics. +Use query plans to understand how LanceDB reads data, applies filters and uses +indexes. `explain_plan()` shows the planned operations. `analyze_plan()` runs +the query and adds measurements, including execution time and rows processed. - -The SQL equivalent is [`EXPLAIN`](/sql/dql/explain). - +## Inspect a plan or measure execution {#inspect-a-query-plan} -## What it shows {#inspect-a-query-plan} +| Operation | Python / Rust | TypeScript | +|:----------|:--------------|:-----------| +| Inspect the plan without executing the query | `explain_plan()` | `explainPlan()` | +| Execute the query and report runtime measurements | `analyze_plan()` | `analyzePlan()` | -Reveals the logical query plan before execution, helping you identify potential issues with query structure and index usage. This tool is useful for: +Use the first to inspect query structure and the second to investigate work +actually performed. Operator names and available measurements can differ +between OSS and Enterprise. SQL provides [EXPLAIN and EXPLAIN ANALYZE](/sql/dql/explain). -- Verifying query optimization strategies -- Validating index selection -- Understanding query execution order -- Detecting missing indices + -Metadata filters are prefiltered by default, which usually shows the filter pushed into the -`LanceScan` or index scan. If you set `prefilter=False`, expect a separate `FilterExec` after -search instead; that can be useful for some expensive filters, but it changes both latency and -the number of rows available after filtering. + -## Reading the Execution Plan {#reading-the-execution-plan} +These examples use three sample records and hand-written vectors, so no +embedding service is needed. Each program creates a temporary local table +and removes it when finished. -To demonstrate query performance analysis, we'll use a table containing 1.2M rows sampled from the [Wikipedia dataset](https://huggingface.co/datasets/wikimedia/wikipedia). Initially, the table has no indices, allowing us to observe the impact of optimization. +- **Python:** Install with `pip install lancedb`, save as `query_plan.py`, then run `python query_plan.py`. +- **TypeScript:** Install with `npm install @lancedb/lancedb`, save as `query_plan.mts`, then run `node query_plan.mts` with Node.js 22.18 or later. +- **Rust:** Download and extract the [complete Rust examples](/static/code/search-rust.zip), then run `cargo run --locked --bin query_plan` from the extracted directory. + + + +{PyQueryPlan} + + +{TsQueryPlanBody + "\n" + TsQueryPlanTail} + + +{RsQueryPlanBody + "\n" + RsQueryPlanTail} + + + +This tiny table demonstrates the API, not a performance benchmark. The plan +uses a scan because the example has no vector index. The following larger +example illustrates how to read scans, filters and index operations. + + + +## Read scans, filters and result retrieval {#reading-the-execution-plan} + +The recorded example below uses 1.2 million rows sampled from the +[Wikipedia dataset](https://huggingface.co/datasets/wikimedia/wikipedia). +The counts describe that run. The table initially has no indexes, so the plan +shows a scan. This Enterprise plan includes `RemoteTake` and hides storage paths +as `uri=***`. A local plan shows the dataset path instead, and fetches the result +rows with Lance's `Take` or, depending on the library version, a second +`LanceRead` whose `source` is `stream(_rowid)`. Let's examine a vector search query that: @@ -67,6 +99,9 @@ const explainPlan = await table ### Execution Plan Components {#execution-plan-components} + + + The execution plan reveals the sequence of operations performed to execute your query. Let's examine each component: ``` @@ -103,7 +138,7 @@ FilterExec: identifier@1 > 0 AND identifier@1 < 1000000 #### 3\. Vector Search {#3-vector-search} -- Computes L2 (Euclidean) distances between query vector and all vectors that passed the filter +- Computes squared L2 distances between query vector and all vectors that passed the filter ``` KNNVectorDistance: metric=l2 @@ -113,7 +148,7 @@ KNNVectorDistance: metric=l2 - Filters out null distance results - Sorts by distance and takes top 100 results -- Processes in batches of 1024 for optimal memory usage +- Coalesces output into batches targeting 1,024 rows in this plan ``` SortExec: TopK(fetch=100) @@ -126,7 +161,7 @@ CoalesceBatchesExec: target_batch_size=1024 #### 5\. Data Retrieval {#5-data-retrieval} -- `RemoteTake` is a key component of Lance's I/O cache +- `RemoteTake` is the deployment's operator for this step; a local plan shows Lance's `Take` or a `LanceRead` over the selected row IDs, depending on the library version - Handles efficient data retrieval from remote storage locations - Fetches specific rows and columns needed for the final output - Optimizes network bandwidth by only retrieving required data @@ -139,19 +174,21 @@ RemoteTake: columns="vector, identifier, _rowid, _distance, chunk_index, title" - Returns only requested columns and maintains column ordering -```python +```text ProjectionExec: expr=[chunk_index@4 as chunk_index, title@5 as title, identifier@1 as identifier, _distance@3 as _distance] ``` This plan demonstrates a basic search without index optimizations: it performs a full scan and filter before vector search. + + + ## Non-vector queries {#analyzing-non-vector-queries} -`explain_plan` is not vector-specific -- it is available on every query builder, -including FTS and hybrid. The most common reason to read the plan for a -non-vector query is to confirm whether your `where` clause pushed into the scan -(good) or ran as a separate `FilterExec` on top of the search results (often -slower, and a hint that the filter column needs a scalar index). +Query plans also cover filters and full-text search. Python can show both +sub-plans of a hybrid query. Check where a filter runs and how many rows flow +through each operator; a separate `FilterExec` does not by itself establish +that the plan is inefficient or that an index is missing. ### FTS queries {#fts-queries} @@ -176,7 +213,10 @@ const plan = await table ``` -In an indexed FTS plan you should see a `MatchQuery` (or other FTS execution node) reading from the inverted index, with the metadata filter pushed down. If the plan shows a `LanceScan` followed by `FilterExec` over the entire text column, the FTS index either isn't covering the column or the filter isn't using a scalar index — both worth investigating. +Look for the text-index operation and any scan of unindexed rows. A scan can +be needed to include records written after the index was built. Use index +coverage and execution measurements to determine whether that extra work +matters. Check filter placement separately from text-index use. ### Hybrid queries {#hybrid-queries} @@ -197,21 +237,19 @@ print(plan) ``` -Reading the plan tells you the shape. `analyze_plan` executes both sub-queries -and labels them `Vector Search Plan:` and `FTS Search Plan:`, which is how you -see whether the filter pushed into both halves uniformly and which half -dominates latency. +`analyze_plan` executes both sub-queries and labels them `Vector Search Plan:` +and `FTS Search Plan:`. Compare their filter placement and execution measurements. `analyze_plan` executes the query and returns the physical plan annotated with -runtime metrics -- elapsed time, rows produced, bytes read, index comparisons. +runtime metrics such as elapsed time, rows produced, bytes read and index comparisons. Where [`explain_plan`](#inspect-a-query-plan) tells you what a query intends to do, -this tells you what it cost. +analysis shows the work performed. The SQL equivalent is [`EXPLAIN ANALYZE`](/sql/dql/explain). -## What it measures {#analyze-a-query-plan} +## Read execution measurements {#analyze-a-query-plan} Executes the query and provides detailed runtime metrics, including: @@ -220,7 +258,7 @@ Executes the query and provides detailed runtime metrics, including: - Index effectiveness (`_index_comparisons_`, `_indices_loaded_`) - Resource utilization (`_iops_`, `_requests_`) -## Performance Analysis {#performance-analysis} +## Measure the example query {#performance-analysis} Let's use `analyze_plan` to run the query and analyze the query performance, which will help us identify potential bottlenecks: @@ -249,6 +287,9 @@ const analyzePlan = await table ### Performance Metrics Analysis {#performance-metrics-analysis} + + + ``` ProjectionExec: expr=[chunk_index@4 as chunk_index, title@5 as title, identifier@1 as identifier, _distance@3 as _distance], metrics=[output_rows=100, elapsed_compute=1.424µs] RemoteTake: columns="vector, identifier, _rowid, _distance, chunk_index, title", metrics=[output_rows=100, elapsed_compute=175.53097ms, output_batches=1, remote_takes=100] @@ -284,11 +325,14 @@ ProjectionExec: expr=[chunk_index@4 as chunk_index, title@5 as title, identifier - Remote take operation for 100 results - Final projection of required columns + + + ### Distributed metrics on remote tables {#distributed-metrics-on-remote-tables} Enterprise -When you call `analyze_plan` against a LanceDB Enterprise table, the query runs across a pool of workers. By default the returned plan aggregates each operator's metrics into a single value, matching the local single-node output. Pass a `distributed_metrics` mode when you need to see how work was split across workers: +When you call `analyze_plan` against a LanceDB Enterprise deployment that distributes its queries across a pool of workers, the query runs on those workers. By default the returned plan aggregates each operator's metrics into a single value, matching the local single-node output. Pass a `distributed_metrics` mode when you need to see how work was split across workers: | Mode | What it shows | When to use it | | :------------ | :---------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------ | @@ -296,7 +340,9 @@ When you call `analyze_plan` against a LanceDB Enterprise table, the query runs | `per_worker` | Metrics reported separately for each worker that participated in the query. | Diagnosing stragglers or uneven work distribution across the cluster. | | `full` | Both the aggregate summary and the per-worker breakdown. | Deep investigations where you need the totals *and* the per-worker detail in a single output. | -The parameter only affects remote plans — local queries always return the aggregate output regardless of what you pass. +Python exposes this option through `distributed_metrics`. In TypeScript, pass +the same mode to `analyzePlan()` as its `distributedMetrics` argument. Local +Python queries ignore the distributed mode and return local measurements. ```python Python icon="python" @@ -310,30 +356,26 @@ plan = ( print(plan) ``` -```typescript TypeScript icon="square-js" -// Show a per-worker breakdown for a remote query -const plan = await table - .search(queryEmbed) - .where("identifier > 0 AND identifier < 1000000") - .limit(100) - .analyzePlan("per_worker"); -``` + -Use `per_worker` or `full` sparingly — the plan output grows with the size of your worker pool. Stick with the default `aggregate` mode for routine tuning. +Use `per_worker` or `full` sparingly; the plan output grows with the size of your worker pool. Stick with the default `aggregate` mode for routine tuning. -### Key Observations {#key-observations} +### Identify expensive operations {#key-observations} -- Vector search is the primary bottleneck (1,099,508 vector comparisons) -- Significant I/O overhead (1.86GB data read) +- The query computes distances for 1,099,508 vectors; inspect timings to assess its contribution to latency +- The scan reads 1.86 GB in this recorded run - Full table scan due to lack of indices -- Substantial optimization potential through proper index implementation +- Compare this scan with an indexed plan using the same query -## Optimized Query Execution {#optimized-query-execution} +## Compare the query after adding indexes {#optimized-query-execution} -After creating vector and scalar indices, the execution plan shows: + + + +The recorded query after adding vector and scalar indexes produced this plan: ``` ProjectionExec: expr=[chunk_index@3 as chunk_index, title@4 as title, identifier@2 as identifier, _distance@0 as _distance] @@ -442,7 +484,7 @@ ANNSubIndex: - Before: L2 calculations on 1,099,508 vectors - After: - - 99.8% reduction in vector comparisons + - About 97.6% fewer vector comparisons (25,893 versus 1,099,508) - Decreased output batches from 1,076 to 20 #### 3\. Data Retrieval Optimization {#3-data-retrieval-optimization} @@ -455,7 +497,10 @@ RemoteTake: - RemoteTake operation remains consistent -## Performance Optimization Guide {#performance-optimization-guide} + + + +## Choose a change to measure {#performance-optimization-guide} ### 1\. Index Implementation {#1-index-implementation} @@ -470,13 +515,14 @@ RemoteTake: | Data Type | Recommended Index | Use Case | | ----------- | ------------------ | ---------------------------------------- | | Vector | IVF_PQ/IVF_HNSW_SQ/IVF_HNSW_FLAT | Approximate nearest neighbor search | -| Scalar | B-Tree | Range queries and sorting | +| Scalar | B-Tree | Equality and range filters | | Categorical | Bitmap | Multi-value filters and set operations | | `List` | Label_list | Multi-label classification and filtering | Use `table.index_stats()` to monitor index coverage. -A well-optimized table should have `num_unindexed_rows ~ 0`. +Unindexed rows can add scan work. Use their count alongside query measurements +to decide when index maintenance is needed. ### 2\. Query Plan Optimization {#2-query-plan-optimization} @@ -485,10 +531,10 @@ A well-optimized table should have `num_unindexed_rows ~ 0`. | Plan Pattern | Optimization | | ------------------------------------------ | -------------------------------------------- | -| LanceScan with high _bytes_read_ or _iops_ | Add missing index | +| Scan with high bytes read or I/O count | Check index coverage and whether the query can use an index | | | Use `select()` to limit returned columns | | | Check whether the dataset has been compacted | -| Multiple sequential filters | Reorder filter conditions | +| Multiple filters | Check their placement and row counts; compare the plan after changing the query | Regularly analyze your query plans to identify and address performance @@ -498,7 +544,8 @@ optimization efforts. ### 3\. Getting Started with Optimization {#3-getting-started-with-optimization} -For vector search performance: - -- Create ANN index on your vector column(s) as described in the [index guide](/indexing/vector-index/) -- If you often filter by metadata, create [scalar indices](/indexing/scalar-index/) on those columns +Keep the query, data and result limit fixed while changing one factor. Compare +rows processed, bytes read, timings and returned results. For ANN changes, +measure recall too: doing less work is useful only if retrieval quality meets +your application's needs. See [vector indexes](/indexing/vector-index) and +[scalar indexes](/indexing/scalar-index) for index choices. diff --git a/docs/web/search/filtering.mdx b/docs/web/search/filtering.mdx index d5bce618b..50d37acc0 100644 --- a/docs/web/search/filtering.mdx +++ b/docs/web/search/filtering.mdx @@ -1,28 +1,103 @@ --- title: "Metadata Filtering in LanceDB" sidebarTitle: Filtering -description: Filter search results in LanceDB based on metadata fields. +description: "Use predicates to select rows and control when filters apply to search." icon: "filter" +availability: + oss: available + enterprise: available + summary: >- + Metadata filtering is available in OSS and Enterprise. --- -LanceDB supports filtering features of query results based on metadata fields. -While joint vector and metadata search at scale presents a significant challenge, -LanceDB achieves sub-100ms latency at thousands of QPS, enabling efficient vector search -with filtering capabilities even on datasets containing billions of records. +import { PyFilterSearch, TsFilterSearchBody, TsFilterSearchTail } from '/snippets/search_examples.mdx'; +import { RsExampleBody as RsFilterSearchBody, RsExampleTail as RsFilterSearchTail } from '/snippets/search_filter_search.mdx'; -**Pre-filtering** means LanceDB applies the metadata `where(...)` condition before running vector search, so the search only considers rows that already match the filter. **Post-filtering** means LanceDB runs vector search first and only then filters the returned candidates. Pre-filtering is enabled by default. In practice, pre-filtering is better when the filter is part of the result contract; post-filtering can be lower-latency for expensive or non-indexable filters, but it can return fewer than `limit` rows, or even zero, if the nearest neighbors do not pass the filter. +A filter restricts results to rows that satisfy a condition. Use it on its +own to select records, or combine it with vector, full-text or hybrid search. +For example, search for similar documents only within a particular category. -On hybrid queries, the same `where(...)` filter is applied to both the vector and full-text halves of the query. The prefilter or postfilter choice controls whether that happens before each subquery scores candidates or after the subquery top-k is produced. + - -The same predicates work in SQL -- see [SELECT](/sql/dql/select#filtering) and [Operators](/sql/operators). - + -## Chaining `where` clauses {#chaining-where-clauses} +These examples use three sample records and hand-written vectors, so no +embedding service is needed. Each program creates a temporary local table +and removes it when finished. -In more recent LanceDB SDK versions (see the callout box below for the exact version numbers), you can call `where(...)` (Python and TypeScript) or `only_if(...)` (Rust) more than once on the same query builder. Each additional filter is combined with the previous one using logical `AND`, so `where("a > 0").where("b < 10")` is equivalent to `where("(a > 0) AND (b < 10)")`. +- **Python:** Install with `pip install lancedb`, save as `filtered_search.py`, then run `python filtered_search.py`. +- **TypeScript:** Install with `npm install @lancedb/lancedb`, save as `filtered_search.mts`, then run `node filtered_search.mts` with Node.js 22.18 or later. +- **Rust:** Download and extract the [complete Rust examples](/static/code/search-rust.zip), then run `cargo run --locked --bin filter_search` from the extracted directory. -For a fixed predicate, writing one `where(...)` clause with `AND` is just as valid and often clearer. Chaining is mainly useful when code composes filters incrementally, such as applying a shared base predicate in a helper and then adding a per-call predicate at the query site. + + +{PyFilterSearch} + + +{TsFilterSearchBody + "\n" + TsFilterSearchTail} + + +{RsFilterSearchBody + "\n" + RsFilterSearchTail} + + + +The query vector is closest to the gardening record, but the filter excludes +that category. The results contain only the two animal records. + + + +## Prefiltering and postfiltering {#choose-when-the-filter-runs} + +- **Prefiltering**, the default, limits the rows eligible for retrieval before + search chooses its candidates. +- **Postfiltering** removes nonmatching rows after candidates are selected. + It can return fewer than `limit` rows, including zero, even when matching + records exist elsewhere in the table. + +Both modes exclude rows that fail the predicate. They differ in which matching +rows the search can find. On hybrid queries, the filter applies to both the +vector and full-text searches. + +SDK filters use SQL expressions without requiring a SQL connection. For +Enterprise SQL queries, see [SELECT](/sql/dql/select#filtering). + +## Filtering examples by language {#example-metadata-filtering} + +The following excerpts use a connected `db` handle. Create this sample table +before running the queries beneath it; each query uses the same four records. + + +```python Python icon="python" +data = [ + {"vector": [3.1, 4.1], "item": "foo", "price": 10.0}, + {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}, + {"vector": [10.2, 100.8], "item": "baz", "price": 30.0}, + {"vector": [1.4, 9.5], "item": "fred", "price": 40.0}, +] +table = db.create_table("metadata_filter_example", data=data, mode="overwrite") +``` + +```typescript TypeScript icon="square-js" +const data = [ + { vector: [3.1, 4.1], item: "foo", price: 10.0 }, + { vector: [5.9, 26.5], item: "bar", price: 20.0 }, + { vector: [10.2, 100.8], item: "baz", price: 30.0 }, + { vector: [1.4, 9.5], item: "fred", price: 40.0 }, +]; + +const tableName = "metadata_filter_example"; +const table = await db.createTable(tableName, data, { + mode: "overwrite", +}); +``` + + +## Combine predicates {#chaining-where-clauses} + +Repeated filter calls combine predicates with logical `AND`. For example, `where("a > 0").where("b < 10")` is equivalent to `where("(a > 0) AND (b < 10)")`. + +Use a single expression for a fixed filter. Chain calls when your application +builds the filter incrementally. ```python Python icon="python" @@ -51,36 +126,6 @@ const result = await table In Python SDK versions before `0.34.0` and Rust/TypeScript SDK versions before `0.31.0`, a second `where(...)` or `only_if(...)` call replaced the first filter, so only the last predicate was applied. If your code needs to run on those older versions, write a single predicate with `AND` instead of chaining calls. When upgrading to the latest SDKs, review any existing chained filters and drop earlier calls you no longer want to apply. -## Example: Metadata Filtering {#example-metadata-filtering} - -To illustrate filtering capabilities, let's try four data points with combinations of vectors and metadata: - - -```python Python icon="python" -data = [ - {"vector": [3.1, 4.1], "item": "foo", "price": 10.0}, - {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}, - {"vector": [10.2, 100.8], "item": "baz", "price": 30.0}, - {"vector": [1.4, 9.5], "item": "fred", "price": 40.0}, -] -table = db.create_table("metadata_filter_example", data=data, mode="overwrite") -``` - -```typescript TypeScript icon="square-js" -const data = [ - { vector: [3.1, 4.1], item: "foo", price: 10.0 }, - { vector: [5.9, 26.5], item: "bar", price: 20.0 }, - { vector: [10.2, 100.8], item: "baz", price: 30.0 }, - { vector: [1.4, 9.5], item: "fred", price: 40.0 }, -]; - -const tableName = "metadata_filter_example"; -const table = await db.createTable(tableName, data, { - mode: "overwrite", -}); -``` - - ### Filtering Without Vector Search {#filtering-without-vector-search} You can always filter your data without search. This is useful when you need to query based on metadata: @@ -104,9 +149,8 @@ const filteredResult = await table ``` - -If your table is large, this could potentially return a very large amount of data. Please be sure to use a `limit` clause unless you're sure you want to return the whole result set. - +Set `limit()` explicitly to control how many matching records your application +receives. ### Pre-Filtering with Vector Search {#pre-filtering-with-vector-search} @@ -124,6 +168,7 @@ filtered_result = ( const results = await table .search([100, 102]) .where("(item IN ('foo', 'bar')) AND (price > 15.0)") + .limit(3) .toArray(); ``` @@ -141,7 +186,7 @@ post_filtered_result = ( ``` ```typescript TypeScript icon="square-js" -const postFilteredResult = await (table.search([100, 102]) as VectorQuery) +const postFilteredResult = await table.vectorSearch([100, 102]) .where("(item IN ('foo', 'bar')) AND (price > 15.0)") .postfilter() .limit(3) @@ -149,14 +194,13 @@ const postFilteredResult = await (table.search([100, 102]) as VectorQuery) ``` - -When querying large tables, omitting a `limit` clause may overwhelm resources and return excessive data. It's always recommended -to be mindful of the potential impact on performance and costs when working with really large tables. - +Postfiltering can return fewer than the requested number of records because +the predicate removes candidates after search. -## Filtering with SQL {#filtering-with-sql} +## Write SQL predicates {#filtering-with-sql} -Because it's built on top of DataFusion, LanceDB embraces the utilization of standard SQL expressions as predicates for filtering operations. SQL can be used during vector search, update, and deletion operations. +Pass a SQL expression as the filter string. These expressions also select rows +for SDK update and delete operations; they are not standalone SQL statements. LanceDB supports a growing list of SQL expressions: @@ -178,7 +222,7 @@ For example, the following filter string is acceptable: ```python Python icon="python" -tbl.search([100, 102]).where( +table.search([100, 102]).where( "(item IN ('foo', 'baz')) AND (price > 20.0)" ).to_arrow() ``` @@ -202,9 +246,10 @@ path must be wrapped in backticks. AND `nested with space`.`inner with space` < 2 ``` - -Field names containing periods (.) are NOT supported. - + +Periods separate nested fields. LanceDB does not allow a literal period in a +top-level column name; backticks do not remove that restriction. + ### Dates, Timestamps, Decimals {#dates-timestamps-decimals} @@ -231,7 +276,8 @@ parameter. Microsecond precision (6) is the default. ## Apache Arrow Mapping {#apache-arrow-mapping} -LanceDB internally stores data in [Apache Arrow](https://arrow.apache.org/) format. +LanceDB uses Arrow types for schemas and in-memory data. On disk, tables use +the [Lance format](/basics/lance). The mapping from SQL types to Arrow types is: | SQL type | Arrow type | @@ -245,26 +291,27 @@ The mapping from SQL types to Arrow types is: | `double` | `Float64` | | `decimal(precision, scale)` | `Decimal128` | | `date` | `Date32` | -| `timestamp` | `Timestamp` [^1] | +| `timestamp` | `Timestamp` | | `string` | `Utf8` | | `binary` | `Binary` | -## Best Practices {#best-practices} +## Speed up frequently used filters {#best-practices} -**Scalar Indexes**: We strongly recommend creating scalar indices on columns used for filtering, whether combined with a search operation or applied independently (e.g., for updates or deletions). +Consider a [scalar index](/indexing/scalar-index) on frequently filtered columns. +Use the query plan to check whether a predicate uses that index. Functions or +casts applied to a column can change which optimizations are available. +For list columns, a label-list index supports membership filters such as +`array_has_any` and `array_has_all`. -For best performance with large tables or high query volumes: +## Understand missing matches {#limitations} -- Build a scalar index on frequently filtered columns -- Use exact column names in filters (e.g., `user_id` instead of `USER_ID`) -- Avoid complex transformations in filter expressions (keep them simple) -- When running concurrent queries, use connection pooling for better throughput +A restrictive predicate can legitimately leave fewer rows than the requested +limit. Approximate vector search can also miss qualifying neighbors, and +postfiltering can discard candidates without retrieving replacements. -For a column of type LIST(T), you can use `LABEL_LIST` to create a scalar index. Then you should leverage DataFusion's [array functions](https://datafusion.apache.org/user-guide/sql/scalar_functions.html#array-functions) like `array_has_any` or `array_has_all` for optimized filtering. - -## Limitations {#limitations} - -Both **pre-filtering** and **post-filtering** can yield false positives. For pre-filtering, if the filter is too selective, it might eliminate relevant items that the vector search would have otherwise identified as a good match. In this case, increasing `nprobes` parameter will help reduce such false positives. It is recommended to call `bypass_vector_index()` if you know that the filter is highly selective. - -Similarly, a highly selective post-filter can lead to false positives. Increasing both `nprobes` and `refine_factor` can mitigate this issue. When deciding between pre-filtering and post-filtering, pre-filtering is generally the safer choice if you're uncertain. +For a recall comparison, run the same query and predicate while bypassing the +vector index on a suitably sized test table. Then compare the indexed query's +results and latency while changing probe or refinement settings. Increasing +those settings does not guarantee a full result set. See +[vector search](/search/vector-search) and [query plans](/search/explain). diff --git a/docs/web/search/full-text-search.mdx b/docs/web/search/full-text-search.mdx index 0008183dc..0dc0096f7 100644 --- a/docs/web/search/full-text-search.mdx +++ b/docs/web/search/full-text-search.mdx @@ -1,8 +1,13 @@ --- title: Full-Text Search (FTS) sidebarTitle: Full-Text Search (FTS) -description: Learn how to implement full-text search in LanceDB using BM25 for keyword-based retrieval. +description: "Find records by words, phrases or partial matches using a full-text index." icon: "book" +availability: + oss: available + enterprise: available + summary: >- + Full-text search is available in OSS and Enterprise. --- import { @@ -12,17 +17,51 @@ import { } from '/snippets/search.mdx' import { PyFtsBooleanQueries } from '/snippets/doc_function_examples.mdx' -LanceDB provides support for Full-Text Search via Lance, allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions. +import { PyTextSearch, TsTextSearchBody, TsTextSearchTail } from '/snippets/search_examples.mdx'; +import { RsExampleBody as RsTextSearchBody, RsExampleTail as RsTextSearchTail } from '/snippets/search_text_search.mdx'; - -In SQL, full-text search is the [`fts()`](/sql/functions#fts) table function, or the [`contains_tokens`](/sql/scalar-functions#contains-tokens) predicate when you do not need a score. - +Full-text search finds documents by matching words and ranks them with BM25. +Use it to search for names, product codes, technical terms or phrases in your +text columns. Create an FTS index on the column before running a query. -## Basic Usage {#basic-usage} +You can also combine keyword matches with vector similarity using +[hybrid search](/search/hybrid-search). -Consider that we have a LanceDB table named `my_table`, whose string column `text` we want to index and query via keyword search, the FTS index must be created before you can search via keywords. + -### Table Setup {#table-setup} + + +Each example creates a temporary local table, runs the query and removes the +table when finished. No embedding model is needed for a text query. + +- **Python:** Install with `pip install lancedb`, save as `text_search.py`, then run `python text_search.py`. +- **TypeScript:** Install with `npm install @lancedb/lancedb`, save as `text_search.mts`, then run `node text_search.mts` with Node.js 22.18 or later. +- **Rust:** Download and extract the [complete Rust examples](/static/code/search-rust.zip), then run `cargo run --locked --bin text_search` from the extracted directory. + + + +{PyTextSearch} + + +{TsTextSearchBody + "\n" + TsTextSearchTail} + + +{RsTextSearchBody + "\n" + RsTextSearchTail} + + + +Each example searches the indexed `text` column and returns the puppy record. +Higher `_score` values rank first; they are relevance scores, not probabilities. + + + +## Basic usage {#basic-usage} + +The steps below create a table, index its `text` column and search for a word. +For a standalone program with imports and sample data, expand the complete +example above. + +### Create the sample table {#table-setup} First, open or create the table you want to search: @@ -66,19 +105,16 @@ let tbl = db ``` -### Construct FTS Index {#construct-fts-index} +### Build the text index {#construct-fts-index} Create a full-text search index on your text column: - -In Python, this page shows the synchronous `create_fts_index(...)` form. For the -asynchronous equivalent (`await table.create_index("text", config=FTS(...))`), see -[FTS index](/indexing/fts-index). - +For asynchronous Python index creation, use +`await table.create_index("text", config=FTS())`. See [FTS index](/indexing/fts-index). ```python Python icon="python" -table.create_fts_index("text") +table.create_index("text", config=FTS()) ``` ```typescript TypeScript icon="square-js" @@ -95,17 +131,24 @@ tbl ``` -### Full-text Search {#full-text-search} +OSS builds the index before the call returns. Enterprise builds it in the +background. Wait for completion with Python’s `table.wait_for_index(["text_idx"])` +or TypeScript’s `await tbl.waitForIndex(["text_idx"], 300)` before querying it. +Use your actual index name if you chose a different one. + +### Run the query {#full-text-search} Perform full-text search and retrieve results: ```python Python icon="python" -results = table.search("puppy") +results = ( + table.search("puppy", query_type="fts", fts_columns="text") .limit(10) - .select(["text"]) + .select(["text", "_score"]) .to_list() -# [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}] +) +# Returns the puppy record with its BM25 score. ``` ```typescript TypeScript icon="square-js" @@ -127,19 +170,26 @@ let results = tbl ``` -The search is conducted on all indexed columns by default, so it's useful when there are multiple indexed columns. - -If you want to specify which columns to search use `fts_columns="text"` +Specify the text column with `fts_columns="text"` in synchronous Python, +especially when the table contains multiple text indexes. -LanceDB automatically searches on the existing FTS index if the input to the search is of type `str`. If you provide a vector as input, LanceDB will search the ANN index instead. +A string query can also invoke an embedding function when the table has +embedding metadata. Use `query_type="fts"` in Python when you specifically +want full-text search. If a table has more than one FTS index, specify the indexed text column in the query. In Python you can use `fts_columns` or the query builder's `nearest_to_text(..., columns=...)`; in TypeScript, use `query().nearestToText(..., columns)`. The newer Lance-native FTS does not accept legacy Tantivy-only index parameters. +BM25 scores depend on the indexed corpus and its term statistics. Compare +rankings on the dataset you will search; treat scores as relative relevance +values rather than probabilities. + ### Keeping the index up to date {#keeping-the-index-up-to-date} -Rows you add after building an FTS index aren't part of the index until you optimize the table. Until then, queries fall back to a flat scan over the unindexed fragments to keep results complete, which slows them down as the unindexed tail grows. Call `table.optimize()` to fold new rows into the existing index — it's the same operation used for vector indexes: +In OSS, new rows can remain outside the FTS index until you optimize the table. +Search includes those rows by scanning them, but that work can increase query +latency. Call `optimize()` to update the index: @@ -157,13 +207,35 @@ tbl.optimize(OptimizeAction::All).await?; ``` -A useful rule of thumb is to call `optimize()` after roughly 100,000 row changes or 20 data-modification operations, whichever comes first. For tables with continuous ingest, schedule it on a cadence that keeps `num_unindexed_rows` (from `table.index_stats(...)`) close to zero. If you want to skip the flat scan over unindexed rows entirely — for example, on a hot read path where stale results are acceptable — call `.fast_search()` on the query so the search returns only indexed results. +Choose an OSS maintenance schedule based on write volume and query latency. +Inspect index coverage with `table.index_stats(...)` and optimize when the +unindexed data adds too much work. `.fast_search()` skips unindexed rows, so +use it only when omitting recent records is acceptable. -## Advanced Usage {#advanced-usage} +Enterprise manages index updates automatically. You do not call `optimize()` +on an Enterprise table. + +## Choose the matching behavior + +| You need | Use | +|:---------|:----| +| Words in any order | A terms query | +| Words in a particular sequence | A [phrase query](#phrase-vs-terms-queries) and an index with token positions | +| Tolerance for typing errors | [Fuzzy search](#fuzzy-search) | +| Parts of words | An [n-gram index](#search-for-substring) | +| Required, optional or excluded terms | [Boolean queries](#boolean-queries) | + +Use a metadata predicate to restrict qualifying records. See +[filtering options](#filtering-options) for prefiltering and postfiltering. +Enterprise SQL provides [`fts()`](/sql/functions#fts) for scored text retrieval. + +## Configure tokenization and matching {#advanced-usage} ### Tokenize Table Data {#tokenize-table-data} -By default, the text is tokenized by splitting on punctuation and whitespaces, and would filter out words that are longer than 40 characters. All words are converted to lowercase. +The default tokenizer splits text at whitespace and punctuation, lowercases +tokens and removes tokens longer than 40 characters. Stemming, stop-word +removal and accent folding are also enabled by default. Stemming is useful for improving search results by reducing words to their root form, e.g. "running" to "run". LanceDB supports stemming for Arabic, Danish, Dutch, English, Finnish, French, German, Greek, Hungarian, Italian, Norwegian, Portuguese, Romanian, Russian, Spanish, Swedish, Tamil, and Turkish. You should set the `base_tokenizer` parameter rather than `tokenizer_name` because you cannot customize the tokenizer if `tokenizer_name` is specified. @@ -177,7 +249,8 @@ table.create_fts_index("text", language="English", replace=True) ``` -The tokenizer is customizable, you can specify how the tokenizer splits the text, and how it filters out words, etc. +Configure tokenization when you create the index. Rebuild the index to change +how existing text is tokenized. **Default index parameters:** - `base_tokenizer`: `"simple"` @@ -188,7 +261,7 @@ The tokenizer is customizable, you can specify how the tokenizer splits the text - `stem`: true - `remove_stop_words`: true - `ascii_folding`: true -- `custom_stop_words`: `None` — pass a `list[str]` to drop additional words beyond the language defaults. Requires `remove_stop_words=True`. +- `custom_stop_words`: `None`; pass a `list[str]` to drop additional words beyond the language defaults. Requires `remove_stop_words=True`. For multilingual use cases, use `base_tokenizer="icu"` for unicode-aware word segmentation on mixed-language text. ICU stands for [International Components for Unicode](https://icu.unicode.org/). The ICU tokenizer uses bundled ICU4X segmenter data, so it does not require external tokenizer model files. It is a good default when documents mix languages or include scripts where the simple tokenizer would keep an unspaced span as one large token. @@ -215,11 +288,15 @@ export LANCE_LANGUAGE_MODEL_HOME=/path/to/lance/language_models For example, `jieba/default` is resolved under `/jieba/default/...`, `lindera/ipadic` under `/lindera/ipadic/...`, and `lindera/ko-dic` under `/lindera/ko-dic/...`. +Model files must be installed where indexing and queries run: on your machine +for OSS, or on the Enterprise cluster for a remote table. + Built-in stop-word removal supports Danish, Dutch, English, Finnish, French, German, Hungarian, Italian, Norwegian, Portuguese, Russian, Spanish, and Swedish. If you use another stemming language, such as Arabic, Greek, Romanian, Tamil, or Turkish, set `remove_stop_words=False` or pass `custom_stop_words`. -For example, for language with accents, you can specify the tokenizer to use `ascii_folding` to remove accents, e.g. 'é' to 'e': +Use `ascii_folding=True` to match accented and unaccented forms, such as +“é” and “e”: ```python Python icon="python" @@ -235,9 +312,9 @@ table.create_fts_index( ### Filtering Options {#filtering-options} -LanceDB full text search supports to filter the search results by a condition, both pre-filtering and post-filtering are supported. - -This can be invoked via the familiar `where` syntax. +Use `where(...)` to restrict text results to rows that match a predicate. +Prefiltering applies the restriction before retrieval; postfiltering removes +nonmatching candidates afterward and can leave fewer results. With pre-filtering: @@ -308,15 +385,17 @@ Lance-based FTS doesn't support queries using boolean operators `OR`, `AND` in t For full-text search you can specify either a **phrase** query like `"the old man and the sea"`, or a **terms** search query like `old man sea`. -To search for a phrase, the index must be created with `with_position=True` and `remove_stop_words=False`: +Enable token positions to search for phrases. Set `remove_stop_words=False` if you also need to match common words such as “the” and “and”: ```python Python icon="python" -table.create_fts_index("text", with_position=True, replace=True) +from lancedb.index import FTS + +table.create_index("text", config=FTS(with_position=True, remove_stop_words=False), replace=True) ``` -This will allow you to search for phrases, but it will also significantly increase the index size and indexing time. +Storing token positions increases the index size and adds indexing work. ### Fuzzy Search {#fuzzy-search} @@ -326,7 +405,7 @@ to find similar terms within a specified edit distance. | Parameter | Type | Default | Description | | -------------- | ---- | ------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | -| fuzziness | int | 0 | Maximum edit distance allowed for each term. If not specified, automatically set based on term length: 0 for length ≤ 2, 1 for length ≤ 5, 2 for length > 5 | +| fuzziness | int | 0 | Maximum edit distance allowed for each term. The default, `0`, requires an exact match; set `1` or `2` to allow typos. | | max_expansions | int | 50 | Maximum number of terms to consider for fuzzy matching. Higher values may improve recall but increase search time | For a complete walkthrough that creates a sample table and demonstrates fuzzy search and relevance boosting, see [the worked example below](#fuzzy-search-and-boosting-example). @@ -344,65 +423,41 @@ LanceDB supports searching for substrings in the text column, you can set the `b ## Fuzzy Search and Boosting Example {#fuzzy-search-and-boosting-example} + + + ### Generate Data {#generate-data} -First, let's create a table with sample text data for testing fuzzy search: +Run the following steps in order. They use a temporary local database with four records, so the same data is available for each query. ```python Python icon="python" +from tempfile import TemporaryDirectory + import lancedb -import numpy as np import pandas as pd -import random +from lancedb.index import FTS +from lancedb.query import MatchQuery, PhraseQuery, BoostQuery, MultiMatchQuery -# Connect to LanceDB -db = lancedb.connect( - uri="db://your-project-slug", - api_key="your-api-key", - region="us-east-1" -) - -# Generate 100 rows of random " " text -table_name = "fts-fuzzy-boosting-test" -vectors = [np.random.randn(128) for _ in range(100)] -verbs = ("runs", "hits", "jumps", "drives", "barfs") -adv = ("crazily.", "dutifully.", "foolishly.", "merrily.", "occasionally.") -adj = ("adorable", "clueless", "dirty", "odd", "stupid") - -def sentence(nouns): - return " ".join(random.choice(words) for words in (nouns, verbs, adv, adj)) - -text = [sentence(("puppy", "car")) for _ in range(100)] -text2 = [sentence(("rabbit", "girl", "monkey")) for _ in range(100)] -count = [random.randint(1, 10000) for _ in range(100)] +directory = TemporaryDirectory() +db = lancedb.connect(directory.name) +table_name = "fts_examples" +text = ["puppy runs merrily", "car drives crazily", "puppy jumps crazily", "car runs slowly"] +text2 = ["rabbit jumps crazily", "girl runs merrily", "monkey jumps slowly", "rabbit runs merrily"] ``` ```typescript TypeScript icon="square-js" -import * as lancedb from "@lancedb/lancedb" +import * as lancedb from "@lancedb/lancedb"; +import { Index, MatchQuery, PhraseQuery, BoostQuery, MultiMatchQuery, BooleanQuery, Occur } from "@lancedb/lancedb"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; -const db = await lancedb.connect({ - uri: "db://your-project-slug", - apiKey: "your-api-key", - region: "us-east-1" -}); - -// Generate 100 rows of random " " text -const tableName = "fts-fuzzy-boosting-test-ts"; -const n = 100; -const verbs = ["runs", "hits", "jumps", "drives", "barfs"]; -const adverbs = ["crazily", "dutifully", "foolishly", "merrily", "occasionally"]; -const adjectives = ["adorable", "clueless", "dirty", "odd", "stupid"]; - -const pick = (words: string[]) => words[Math.floor(Math.random() * words.length)]; -const sentence = (nouns: string[]) => - [nouns, verbs, adverbs, adjectives].map(pick).join(" "); - -const vectors = Array.from({ length: n }, () => - Array.from({ length: 128 }, () => Math.random() * 2 - 1) -); -const text = Array.from({ length: n }, () => sentence(["puppy", "car"])); -const text2 = Array.from({ length: n }, () => sentence(["rabbit", "girl", "monkey"])); -const count = Array.from({ length: n }, () => Math.floor(Math.random() * 10000) + 1); +const directory = await mkdtemp(join(tmpdir(), "lancedb-fts-")); +const db = await lancedb.connect(directory); +const tableName = "fts_examples"; +const text = ["puppy runs merrily", "car drives crazily", "puppy jumps crazily", "car runs slowly"]; +const text2 = ["rabbit jumps crazily", "girl runs merrily", "monkey jumps slowly", "rabbit runs merrily"]; ``` @@ -414,11 +469,9 @@ const count = Array.from({ length: n }, () => Math.floor(Math.random() * 10000) table = db.create_table( table_name, data=pd.DataFrame({ - "vector": vectors, - "id": [i % 2 for i in range(100)], + "id": range(len(text)), "text": text, "text2": text2, - "count": count, }), mode="overwrite" ) @@ -426,15 +479,7 @@ table = db.create_table( ```typescript TypeScript icon="square-js" // Create table with sample data -const data = makeArrowTable( - vectors.map((vector, i) => ({ - vector, - id: i % 2, - text: text[i], - text2: text2[i], - count: count[i], - })) -); +const data = text.map((value, i) => ({ id: i, text: value, text2: text2[i] })); const table = await db.createTable(tableName, data, { mode: "overwrite" }); ``` @@ -447,14 +492,14 @@ Create a full-text search index on the first text column: ```python Python icon="python" # Create FTS index on first text column -table.create_fts_index("text") -wait_for_index(table, "text_idx") +table.create_index("text", config=FTS(with_position=True, remove_stop_words=False)) +table.wait_for_index(["text_idx"]) ``` ```typescript TypeScript icon="square-js" // Create FTS index on first text column -await table.createIndex("text", { config: Index.fts() }); -await waitForIndex(table, "text_idx"); +await table.createIndex("text", { config: Index.fts({ withPosition: true, removeStopWords: false }) }); +await table.waitForIndex(["text_idx"], 300); ``` @@ -463,14 +508,14 @@ Then, create an index on the second text column: ```python Python icon="python" # Create FTS index on second text column -table.create_fts_index("text2") -wait_for_index(table, "text2_idx") +table.create_index("text2", config=FTS(with_position=True, remove_stop_words=False)) +table.wait_for_index(["text2_idx"]) ``` ```typescript TypeScript icon="square-js" // Create FTS index on second text column -await table.createIndex("text2", { config: Index.fts() }); -await waitForIndex(table, "text2_idx"); +await table.createIndex("text2", { config: Index.fts({ withPosition: true, removeStopWords: false }) }); +await table.waitForIndex(["text2_idx"], 300); ``` @@ -487,19 +532,17 @@ from lancedb.query import MatchQuery # Basic match (exact search) basic_match_results = ( table.search(MatchQuery("crazily", "text")) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) - .to_pandas() + .to_arrow().to_pandas() ) ``` ```typescript TypeScript icon="square-js" -import { MatchQuery } from "@lancedb/lancedb"; - // Basic match (exact search) const basicMatchResults = await table.query() .fullTextSearch(new MatchQuery("crazily", "text")) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) .toArray(); ``` @@ -512,9 +555,9 @@ const basicMatchResults = await table.query() # Fuzzy match (allows typos) fuzzy_results = ( table.search(MatchQuery("craziou", "text", fuzziness=2)) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) - .to_pandas() + .to_arrow().to_pandas() ) ``` @@ -524,34 +567,35 @@ const fuzzyResults = await table.query() .fullTextSearch(new MatchQuery("craziou", "text", { fuzziness: 2, })) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) .toArray(); ``` -#### Prefix based Match {#prefix-based-match} +#### Require an exact prefix in fuzzy matching {#prefix-based-match} -Prefix-based match allows you to search for documents containing words that start with a specific prefix. +`prefix_length` keeps the first characters unchanged when matching a misspelled term. It does not turn a short query into a prefix-completion search. ```python Python icon="python" # Fuzzy match (allows typos) -fuzzy_results = ( - table.search(MatchQuery("cra", "text", prefix_length=3)) - .select(["id", "text"]) +prefix_results = ( + table.search(MatchQuery("craziou", "text", fuzziness=2, prefix_length=3)) + .select(["id", "text", "_score"]) .limit(100) - .to_pandas() + .to_arrow().to_pandas() ) ``` ```typescript TypeScript icon="square-js" // Fuzzy match (allows typos) -const fuzzyResults = await table.query() - .fullTextSearch(new MatchQuery("cra", "text", { +const prefixResults = await table.query() + .fullTextSearch(new MatchQuery("craziou", "text", { + fuzziness: 2, prefixLength: 3, })) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) .toArray(); ``` @@ -580,26 +624,24 @@ from lancedb.query import PhraseQuery phrase_results = ( table.search(PhraseQuery("puppy runs", "text")) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) - .to_pandas() + .to_arrow().to_pandas() ) ``` ```typescript TypeScript icon="square-js" -import { PhraseQuery } from "@lancedb/lancedb"; - // Exact phrase match const phraseResults = await table.query() .fullTextSearch(new PhraseQuery("puppy runs", "text")) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) .toArray(); ``` #### Flexible Phrase Match {#flexible-phrase-match} -To provide more flexible phrase matching, LanceDB supports the `slop` parameter. This allows you to match phrases where the terms appear close to each other, even if they are not directly adjacent or in the exact order, as long as they are within the specified `slop` value. +To provide more flexible phrase matching, LanceDB supports the `slop` parameter. This allows you to match phrases where the terms appear close to each other, with gaps between them. The `slop` value limits the extra positions allowed in the phrase. For example, the phrase query "puppy merrily" would not return any results by default. However, if you set `slop=1`, it will match phrases like "puppy jumps merrily", "puppy runs merrily", and similar variations where one word appears between "puppy" and "merrily". @@ -608,21 +650,19 @@ For example, the phrase query "puppy merrily" would not return any results by de # Flexible phrase match with slop=1 for 'puppy merrily' from lancedb.query import PhraseQuery -phrase_results = ( +slop_results = ( table.search(PhraseQuery("puppy merrily", "text", slop=1)) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) - .to_pandas() + .to_arrow().to_pandas() ) ``` ```typescript TypeScript icon="square-js" -import { PhraseQuery } from "@lancedb/lancedb"; - -// Flexible phrase match with slop=1 for 'puppy runs' -const phraseResults = await table.query() - .fullTextSearch(new PhraseQuery("puppy runs", "text", { slop: 1 })) - .select(["id", "text"]) +// Flexible phrase match with slop=1 for 'puppy merrily' +const slopResults = await table.query() + .fullTextSearch(new PhraseQuery("puppy merrily", "text", { slop: 1 })) + .select(["id", "text", "_score"]) .limit(100) .toArray(); ``` @@ -656,17 +696,17 @@ boosting_results = ( negative_boost=0.2, ), ) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) - .to_pandas() + .to_arrow().to_pandas() ) # Search across both text and text2 multi_match_results = ( table.search(MultiMatchQuery("crazily", ["text", "text2"])) - .select(["id", "text", "text2"]) + .select(["id", "text", "text2", "_score"]) .limit(100) - .to_pandas() + .to_arrow().to_pandas() ) # Search with field boosting @@ -674,28 +714,26 @@ multi_match_boosting_results = ( table.search( MultiMatchQuery("crazily", ["text", "text2"], boosts=[1.0, 2.0]), ) - .select(["id", "text", "text2"]) + .select(["id", "text", "text2", "_score"]) .limit(100) - .to_pandas() + .to_arrow().to_pandas() ) ``` ```typescript TypeScript icon="square-js" -import { MatchQuery, BoostQuery, MultiMatchQuery } from "@lancedb/lancedb"; - // Boosting Example const boostingResults = await table.query() .fullTextSearch(new BoostQuery(new MatchQuery("runs", "text"), new MatchQuery("puppy", "text"), { negativeBoost: 0.2, })) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) .toArray(); // Search across both text fields const multiMatchResults = await table.query() .fullTextSearch(new MultiMatchQuery("crazily", ["text", "text2"])) - .select(["id", "text", "text2"]) + .select(["id", "text", "text2", "_score"]) .limit(100) .toArray(); @@ -704,24 +742,12 @@ const multiMatchBoostingResults = await table.query() .fullTextSearch(new MultiMatchQuery("crazily", ["text", "text2"], { boosts: [1.0, 2.0], })) - .select(["id", "text", "text2"]) + .select(["id", "text", "text2", "_score"]) .limit(100) .toArray(); ``` - -- Use fuzzy search when handling user input that may contain typos or variations -- Apply field boosting to prioritize matches in more important columns -- Combine fuzzy search with boosting for robust and precise search results - -**Recommendations for optimal FTS performance:** - -- Create full-text search indices on text columns that will be frequently searched -- For hybrid search combining text and vectors, see our [hybrid search guide](/search/hybrid-search/) -- For performance benchmarks, check our [benchmark results](/enterprise/benchmarks/) -- For complex queries, use SQL to combine FTS with other filter conditions - ### Boolean Queries {#boolean-queries} LanceDB supports boolean logic in full-text search, allowing you to combine multiple queries using `and` and `or` operators. This is useful when you want to match documents that satisfy multiple conditions (intersection) or at least one of several conditions (union). @@ -734,16 +760,7 @@ In Python, combine two `MatchQuery` objects with `&` for intersection or `|` for {PyFtsBooleanQueries} -In TypeScript, boolean queries are constructed using the `BooleanQuery` class with a list of [Occur, subquery] pairs. For example, to perform an AND query: - -```sql SQL icon="code" -BooleanQuery([ -[Occur.Must, new MatchQuery("puppy", "text")], -[Occur.Must, new MatchQuery("merrily", "text")], -]) -``` - -This approach allows you to specify complex boolean logic by combining multiple subqueries with different Occur values (such as `Must`, `Should`, or `MustNot`). +In TypeScript, pass `[Occur, subquery]` pairs to `BooleanQuery`. Use `Must` for required matches, `Should` for alternatives, and `MustNot` to exclude matches. **Which queries are allowed?** @@ -759,24 +776,22 @@ from lancedb.query import MatchQuery and_query = MatchQuery("puppy", "text") & MatchQuery("merrily", "text") and_results = ( table.search(and_query) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) - .to_pandas() + .to_arrow().to_pandas() ) # Example: Find documents containing either "puppy" or "merrily" or_query = MatchQuery("puppy", "text") | MatchQuery("merrily", "text") or_results = ( table.search(or_query) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) - .to_pandas() + .to_arrow().to_pandas() ) ``` ```typescript TypeScript icon="square-js" expandable=true -import { MatchQuery, BooleanQuery, Occur } from "@lancedb/lancedb"; - // Flexible boolean queries with MatchQuery // Find documents containing both "puppy" and "merrily" @@ -787,7 +802,7 @@ const mustResults = await table [Occur.Must, new MatchQuery("merrily", "text")], ]), ) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) .toArray(); @@ -799,7 +814,7 @@ const shouldResults = await table [Occur.Should, new MatchQuery("merrily", "text")], ]), ) - .select(["id", "text"]) + .select(["id", "text", "_score"]) .limit(100) .toArray(); ``` @@ -808,12 +823,33 @@ const shouldResults = await table **How to use booleans?** -- Use `and`/`&`(Python), `Occur.Must`(Typescript) for intersection (documents must match all queries). -- Use `or`/`|`(Python), `Occur.Should`(Typescript) for union (documents must match at least one query). +- Use `&` (Python), `Occur.Must`(Typescript) for intersection (documents must match all queries). +- Use `|` (Python), `Occur.Should`(Typescript) for union (documents must match at least one query). + +### Clean up the example + +Remove the temporary database when you have finished running the queries. + + +```python Python icon="python" +directory.cleanup() +``` +```typescript TypeScript icon="square-js" +table.close(); +db.close(); +await rm(directory, { recursive: true, force: true }); +``` + + + + ## Substring Search Example {#substring-search-example} + + + LanceDB supports searching for substrings in text columns using n-gram tokenization. This is useful for finding partial matches within text content. ### Setting Up the Table {#setting-up-the-table} @@ -825,11 +861,15 @@ First, create a table with sample text data and configure n-gram tokenization: import pyarrow as pa import lancedb -db = lancedb.connect(":memory:") +from tempfile import TemporaryDirectory +from lancedb.index import FTS + +directory = TemporaryDirectory() +db = lancedb.connect(directory.name) data = pa.table({"text": ["hello world", "lance database", "lance is cool"]}) table = db.create_table("test", data=data) -table.create_fts_index("text", base_tokenizer="ngram") +table.create_index("text", config=FTS(base_tokenizer="ngram")) ``` @@ -868,12 +908,10 @@ You can customize the n-gram behavior by adjusting the minimum length and using ```python Python icon="python" -table.create_fts_index( +table.create_index( "text", - base_tokenizer="ngram", + config=FTS(base_tokenizer="ngram", ngram_min_length=2, prefix_only=True), replace=True, - ngram_min_length=2, - prefix_only=True, ) ``` @@ -899,34 +937,47 @@ assert set(r["text"] for r in results) == {"lance database", "lance is cool"} ``` + +```python Python icon="python" +directory.cleanup() +``` + + + ## Full-Text Search on Array Fields {#full-text-search-on-array-fields} + + + LanceDB supports full-text search on string array columns, enabling efficient keyword-based search across multiple values within a single field (e.g., tags, keywords). ### Setting Up the Connection {#setting-up-the-connection} -Connect to your LanceDB instance: +This example uses a temporary local database. Run the steps in order. ```python Python icon="python" -import lancedb +from tempfile import TemporaryDirectory -# Connect to LanceDB -db = lancedb.connect( - uri="db://your-project-slug", - api_key="your-api-key", - region="us-east-1" -) +import lancedb +import pyarrow as pa +from lancedb.index import FTS +from lancedb.query import MatchQuery, PhraseQuery + +directory = TemporaryDirectory() +db = lancedb.connect(directory.name) ``` -```typescript TypeScript icon="square-js" expandable=true -import * as lancedb from "@lancedb/lancedb" +```typescript TypeScript icon="square-js" +import * as lancedb from "@lancedb/lancedb"; +import { Index, MatchQuery, PhraseQuery, makeArrowTable } from "@lancedb/lancedb"; +import { Schema, Field, List, Utf8 } from "apache-arrow"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; -const db = await lancedb.connect({ - uri: "db://your-project-slug", - apiKey: "your-api-key", - region: "us-east-1" -}); +const directory = await mkdtemp(join(tmpdir(), "lancedb-array-fts-")); +const db = await lancedb.connect(directory); ``` @@ -1048,25 +1099,25 @@ console.log(`Created table: ${tableName}`); ### Building the Full-Text Search Index {#building-the-full-text-search-index} -Create an FTS index on the tags column to enable efficient text search: +Store token positions for phrase queries. This example disables stemming so fuzzy matching compares the original word forms: ```python Python icon="python" # Create FTS index -table.create_fts_index("tags") -wait_for_index(table, "tags_idx") +table.create_index("tags", config=FTS(with_position=True, stem=False)) +table.wait_for_index(["tags_idx"]) ``` ```typescript TypeScript icon="square-js" // Create FTS index console.log("Creating FTS index on 'tags' column..."); await table.createIndex("tags", { - config: Index.fts() + config: Index.fts({ withPosition: true, stem: false }) }); // Wait for index const ftsIndexName = "tags_idx"; -await waitForIndex(table, ftsIndexName); +await table.waitForIndex([ftsIndexName], 300); ``` @@ -1080,24 +1131,27 @@ Search for terms with typos using fuzzy matching: print("\nSearching for 'learning' in tags with a typo:") result = ( table.search(MatchQuery("learnin", column="tags", fuzziness=1)) - .select(['id', 'tags', 'description']) + .select(["id", "tags", "description", "_score"]) .to_arrow() ) ``` -```typescript TypeScript icon="square-js"> +```typescript TypeScript icon="square-js" // Search examples console.log("\nSearching for 'learning' in tags with a typo:"); const fuzzyResults = await table.query() .fullTextSearch(new MatchQuery("learnin", "tags", { - fuzziness: 2, + fuzziness: 1, })) - .select(["id", "tags", "description"]) + .select(["id", "tags", "description", "_score"]) .toArray(); console.log(fuzzyResults); ``` +A string-array query can return a row more than once when several elements match. +Deduplicate by your record ID if your application needs one result per row. + ### Performing Phrase Search {#performing-phrase-search} Search for exact phrases within the array fields: @@ -1107,7 +1161,7 @@ Search for exact phrases within the array fields: print("\nSearching for 'machine learning' in tags:") result = ( table.search(PhraseQuery("machine learning", column="tags")) - .select(['id', 'tags', 'description']) + .select(["id", "tags", "description", "_score"]) .to_arrow() ) ``` @@ -1116,8 +1170,25 @@ result = ( console.log("\nSearching for 'machine learning' in tags:"); const phraseResults = await table.query() .fullTextSearch(new PhraseQuery("machine learning", "tags")) - .select(["id", "tags", "description"]) + .select(["id", "tags", "description", "_score"]) .toArray(); console.log(phraseResults); ``` + +### Clean up the array example + +Remove the temporary database when you have finished running the queries. + + +```python Python icon="python" +directory.cleanup() +``` +```typescript TypeScript icon="square-js" +table.close(); +db.close(); +await rm(directory, { recursive: true, force: true }); +``` + + + diff --git a/docs/web/search/hybrid-search.mdx b/docs/web/search/hybrid-search.mdx index 4a2d2dead..c3e453145 100644 --- a/docs/web/search/hybrid-search.mdx +++ b/docs/web/search/hybrid-search.mdx @@ -1,26 +1,77 @@ --- title: Hybrid Search sidebarTitle: "Hybrid Search" -description: Learn how to perform hybrid search in LanceDB by combining vector and full-text search techniques with reranking. +description: "Combine vector similarity and keyword matches in a single ranked result set." icon: "search" +availability: + oss: available + enterprise: available + summary: >- + Hybrid search is available in OSS and Enterprise. --- -In certain cases, you may want to retrieve documents that are semantically similar to a given query, -but also prioritize specific keywords. This is an example of **hybrid search**, a query method that combines -multiple search techniques. +import { PyConnectEnterprisePlaceholders, TsConnectEnterprisePlaceholders } from '/snippets/connection.mdx'; -For detailed examples, look at this [Python Notebook](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/saas_examples/python_notebook/Hybrid_search.ipynb) or the [**TypeScript Example**](https://github.com/lancedb/vectordb-recipes/tree/main/examples/saas_examples/ts_example/hybrid-search) +import { PyHybridSearch, TsHybridSearchBody, TsHybridSearchTail } from '/snippets/search_examples.mdx'; +import { RsExampleBody as RsHybridSearchBody, RsExampleTail as RsHybridSearchTail } from '/snippets/search_hybrid_search.mdx'; -## Example: Hybrid Search {#example-hybrid-search} +Hybrid search combines vector similarity with keyword matching. It is useful +when a query needs both: for example, finding products with a particular model +number whose descriptions match what the customer is looking for. + +LanceDB runs a vector search and a full-text search, then merges their results +with a reranker. Reciprocal Rank Fusion (RRF) is the default. It combines the +positions of records in the two ranked lists. + +Before you query, store vectors and create an FTS index on the text column. + + + + + +These examples use three sample records and hand-written vectors, so no +embedding service is needed. Each program creates a temporary local table +and removes it when finished. + +- **Python:** Install with `pip install lancedb`, save as `hybrid_search.py`, then run `python hybrid_search.py`. +- **TypeScript:** Install with `npm install @lancedb/lancedb`, save as `hybrid_search.mts`, then run `node hybrid_search.mts` with Node.js 22.18 or later. +- **Rust:** Download and extract the [complete Rust examples](/static/code/search-rust.zip), then run `cargo run --locked --bin hybrid_search` from the extracted directory. + + + +{PyHybridSearch} + + +{TsHybridSearchBody + "\n" + TsHybridSearchTail} + + +{RsHybridSearchBody + "\n" + RsHybridSearchTail} + + + +The puppy record ranks first because both searches place it first. The query +returns `_relevance_score`, with higher values first. This small example shows +the mechanics; evaluate retrieval quality with embeddings and queries from +your own application. + + + +## Generate query embeddings automatically {#example-hybrid-search} ### 1\. Setup {#1-setup} -Import the necessary libraries and dependencies for working with LanceDB, OpenAI embeddings, and reranking. +The following walkthrough uses Sentence Transformers in Python and OpenAI in +TypeScript. Install the corresponding provider packages before starting. +Python downloads a local model; TypeScript calls OpenAI. Install +`lancedb sentence-transformers pandas` for Python, or +`@lancedb/lancedb apache-arrow openai` for TypeScript. Set your OpenAI API key +as described in the [OpenAI embedding setup](/integrations/embedding/openai). +Run the steps for your chosen language in order, choosing one connection option. ```python Python icon="python" -import os import lancedb -import openai +from lancedb.index import FTS +from lancedb.rerankers import RRFReranker from lancedb.embeddings import get_registry from lancedb.pydantic import LanceModel, Vector ``` @@ -33,7 +84,7 @@ import { Utf8 } from "apache-arrow"; ### 2\. Connect to LanceDB {#2-connect-to-lancedb} -Establish a connection to your LanceDB instance, with different options for Enterprise setups or open source. +Choose a local OSS connection or use your Enterprise connection details. OSS @@ -44,9 +95,6 @@ db = lancedb.connect(uri) ``` ```typescript TypeScript icon="square-js" -import * as lancedb from "@lancedb/lancedb"; -import * as arrow from "apache-arrow"; - const databaseDir = "data/sample-lancedb"; const db = await lancedb.connect(databaseDir); ``` @@ -54,41 +102,21 @@ const db = await lancedb.connect(databaseDir); Enterprise -For LanceDB Enterprise, set the `db://` URI, region and the host override to your private cloud endpoint: +Replace the placeholders with your Enterprise database URI, API key, region and +endpoint. In application code, load credentials from environment variables or a +secrets manager. -```python Python icon="python" -host_override = os.environ.get("LANCEDB_HOST_OVERRIDE") - -db = lancedb.connect( - uri=uri, - api_key=api_key, - region=region, - host_override=host_override -) -``` - -```typescript TypeScript icon="square-js" -import * as lancedb from "@lancedb/lancedb"; -import * as arrow from "apache-arrow"; - -const uri = "db://my-lancedb-instance/my-database"; -const apiKey = process.env.LANCEDB_API_KEY; -const region = process.env.LANCEDB_REGION; -const hostOverride = process.env.LANCEDB_HOST_OVERRIDE; - -const db = await lancedb.connect(uri, { - apiKey, - region - hostOverride, -}); -``` + + {PyConnectEnterprisePlaceholders} + + + {TsConnectEnterprisePlaceholders} + - - ### 3\. Configure Embedding Model {#3-configure-embedding-model} -Set up the any embedding model that will convert text into vector representations for semantic search. +Configure the model used for both document and query embeddings. ```python Python icon="python" @@ -108,6 +136,7 @@ Define the data structure for your documents, including both the text content an ```python Python icon="python" class Documents(LanceModel): + category: str text: str = embeddings.SourceField() vector: Vector(embeddings.ndims()) = embeddings.VectorField() @@ -117,6 +146,7 @@ table = db.create_table(table_name, schema=Documents, mode="overwrite") ```typescript TypeScript icon="square-js" const documentSchema = lancedb.embedding.LanceSchema({ + category: new Utf8(), text: embedFunc.sourceField(new Utf8()), vector: embedFunc.vectorField(), }); @@ -134,20 +164,20 @@ Insert sample documents into your table, which will be used for both semantic an ```python Python icon="python" data = [ - {"text": "rebel spaceships striking from a hidden base"}, - {"text": "have won their first victory against the evil Galactic Empire"}, - {"text": "during the battle rebel spies managed to steal secret plans"}, - {"text": "to the Empire's ultimate weapon the Death Star"}, + {"text": "rebel spaceships striking from a hidden base", "category": "film"}, + {"text": "have won their first victory against the evil Galactic Empire", "category": "film"}, + {"text": "during the battle rebel spies managed to steal secret plans", "category": "film"}, + {"text": "to the Empire's ultimate weapon the Death Star", "category": "film"}, ] table.add(data=data) ``` ```typescript TypeScript icon="square-js" const data = [ - { text: "rebel spaceships striking from a hidden base" }, - { text: "have won their first victory against the evil Galactic Empire" }, - { text: "during the battle rebel spies managed to steal secret plans" }, - { text: "to the Empire's ultimate weapon the Death Star" }, + { text: "rebel spaceships striking from a hidden base", category: "film" }, + { text: "have won their first victory against the evil Galactic Empire", category: "film" }, + { text: "during the battle rebel spies managed to steal secret plans", category: "film" }, + { text: "to the Empire's ultimate weapon the Death Star", category: "film" }, ]; await table.add(data); console.log(`Created table: ${tableName} with ${data.length} rows`); @@ -159,8 +189,8 @@ Create a full-text search index on the text column to enable keyword-based searc ```python Python icon="python" -table.create_fts_index("text") -wait_for_index(table, "text_idx") +table.create_index("text", config=FTS()) +table.wait_for_index(["text_idx"]) ``` ```typescript TypeScript icon="square-js" @@ -168,12 +198,13 @@ console.log("Creating full-text search index..."); await table.createIndex("text", { config: lancedb.Index.fts(), }); -await waitForIndex(table as any, "text_idx"); +await table.waitForIndex(["text_idx"], 300); ``` ### 7\. Set Reranker [Optional] {#7-set-reranker-optional} -Initialize the reranker that will combine and rank results from both semantic and keyword search. By default, lancedb uses RRF reranker, but you can choose other rerankers like `Cohere`, `CrossEncoder`, or others lister in integrations section. +RRF is the default. The examples below configure it explicitly. Other +rerankers and their language support are described in [Reranking](/reranking). ```python Python icon="python" @@ -192,7 +223,7 @@ Perform a hybrid search query that combines semantic similarity with keyword mat ```python Python icon="python" results = ( table.search( - "flower moon", + "rebel spaceships", query_type="hybrid", vector_column_name="vector", fts_columns="text", @@ -208,10 +239,10 @@ print(results) ```typescript TypeScript icon="square-js" console.log("Performing hybrid search..."); -const queryVector = await embedFunc.computeQueryEmbeddings("full moon in May"); +const queryVector = await embedFunc.computeQueryEmbeddings("rebel spaceships"); const hybridResults = await table .query() - .fullTextSearch("flower moon") + .fullTextSearch("rebel spaceships") .nearestTo(queryVector) .rerank(reranker) .select(["text"]) @@ -223,13 +254,16 @@ console.log(hybridResults); ``` -### 9\. Hybrid Search - Explicit Vector and Text Query pattern {#9-hybrid-search-explicit-vector-and-text-query-pattern} -You can also pass the vector and text query explicitly. This is useful if you're not using the embedding API or if you're using a separate embedder service. +### 9\. Supply a vector and text separately {#9-hybrid-search-explicit-vector-and-text-query-pattern} +You can supply the vector and text separately. The vector must use the same +model and dimensions as the stored embeddings. This example reuses the model +configured above; the complete example at the top also demonstrates this pattern +in TypeScript and Rust. ```python Python icon="python" -vector_query = [0.1, 0.2, 0.3, 0.4, 0.5] -text_query = "flower moon" +text_query = "rebel spaceships" +vector_query = embeddings.compute_query_embeddings(text_query)[0] ( table.search(query_type="hybrid") .vector(vector_query) @@ -240,13 +274,15 @@ text_query = "flower moon" ``` -## Query controls {#query-controls} +## Control the candidate results {#query-controls} -Hybrid queries inherit the same builder API as vector and FTS queries, so the same knobs for filtering, distance bounds, and row identity apply. These compose with `.rerank(...)` and the explicit `.vector()` / `.text()` form shown above. +Use a result limit, metadata filters and vector distance bounds to control +which candidates are available to the reranker. The examples below show each +option on the query builder. -Always set `.limit(...)` on production hybrid queries. LanceDB's default search limit is 10, but an -explicit cap gives you a clear top-k contract to tune before reranking. +Set `limit()` to the number of results your application needs. The default is 10. +A larger candidate set can help a reranker find relevant records, but adds work. ### Returning row IDs {#returning-row-ids} @@ -256,7 +292,7 @@ Pass `with_row_id(True)` (Python) or `withRowId()` (TypeScript) to include the i ```python Python icon="python" results = ( - table.search("flower moon", query_type="hybrid") + table.search("rebel spaceships", query_type="hybrid") .with_row_id(True) .limit(10) .to_pandas() @@ -267,7 +303,7 @@ results = ( ```typescript TypeScript icon="square-js" const results = await table .query() - .fullTextSearch("flower moon") + .fullTextSearch("rebel spaceships") .nearestTo(queryVector) .withRowId() .limit(10) @@ -282,7 +318,7 @@ const results = await table ```python Python icon="python" results = ( - table.search("flower moon", query_type="hybrid") + table.search("rebel spaceships", query_type="hybrid") .distance_range(lower_bound=0.0, upper_bound=0.4) .limit(10) .to_pandas() @@ -292,7 +328,7 @@ results = ( ```typescript TypeScript icon="square-js" const results = await table .query() - .fullTextSearch("flower moon") + .fullTextSearch("rebel spaceships") .nearestTo(queryVector) .distanceRange(0.0, 0.4) .limit(10) @@ -304,18 +340,21 @@ Either bound can be omitted to leave that side unbounded. ### Prefilter vs. postfilter {#prefilter-vs-postfilter} -When the query carries a metadata filter via `where(...)`, you can choose whether the filter runs before or after the vector and FTS sub-queries. **Prefiltering** (the default) applies `where` to the candidate set before scoring, which is usually what you want — it shrinks the working set and benefits from any scalar indexes on the filter columns. **Postfiltering** runs the filter on the already-ranked top-k from each sub-query; this can be faster when the filter is non-selective or unindexed, but it may return fewer than `limit` rows because some of the top-k may be filtered out. +A hybrid query applies its metadata filter to both the vector and full-text +searches. Prefiltering, the default, restricts eligible rows before retrieval. +Postfiltering removes candidates after retrieval and can leave fewer results +than requested. See [Filtering](/search/filtering). ```python Python icon="python" # Prefilter (default): filter applied before scoring -table.search("flower moon", query_type="hybrid") \ +table.search("rebel spaceships", query_type="hybrid") \ .where("category = 'film'", prefilter=True) \ .limit(10) \ .to_pandas() # Postfilter: filter applied after the sub-queries return top-k -table.search("flower moon", query_type="hybrid") \ +table.search("rebel spaceships", query_type="hybrid") \ .where("category = 'film'", prefilter=False) \ .limit(10) \ .to_pandas() @@ -324,7 +363,7 @@ table.search("flower moon", query_type="hybrid") \ ```typescript TypeScript icon="square-js" // Prefilter (default): just call .where(...) await table.query() - .fullTextSearch("flower moon") + .fullTextSearch("rebel spaceships") .nearestTo(queryVector) .where("category = 'film'") .limit(10) @@ -332,7 +371,7 @@ await table.query() // Postfilter: chain .postfilter() after .where(...) await table.query() - .fullTextSearch("flower moon") + .fullTextSearch("rebel spaceships") .nearestTo(queryVector) .where("category = 'film'") .postfilter() @@ -341,15 +380,20 @@ await table.query() ``` -The choice gets baked into both sub-queries, so the vector and FTS halves see the filter applied the same way. Use [`explain_plan`](/search/explain#analyzing-non-vector-queries) on a hybrid query to see whether the filter pushed into the scan or ran as a separate `FilterExec` step. +In Python, use [`explain_plan`](/search/explain#analyzing-non-vector-queries) on a hybrid query to see whether the filter pushed into the scan or ran as a separate `FilterExec` step. -## More on Reranking {#more-on-reranking} +## Choose and evaluate a reranker {#more-on-reranking} -You can perform hybrid search in LanceDB by combining the results of semantic and full-text search via a reranking algorithm of your choice. LanceDB comes with [**built-in rerankers**](https://docs.lancedb.com/reranking) and you can implement your own **custom reranker** as well. +Start with RRF, then compare alternatives on representative queries. Review +which relevant rows reach the candidate set before evaluating how a reranker +orders them. Record latency as well as relevance: increasing candidate counts +or applying a model-based reranker adds work. -By default, LanceDB uses `RRFReranker()`, which uses reciprocal rank fusion score, to combine and rerank the results of semantic and full-text search. You can customize the hyperparameters as needed or write your own custom reranker. Here's how you can use any of the available rerankers: +Reranking runs in the application for both OSS and Enterprise connections. +See [Reranking](/reranking) for model-free fusion, model-based rerankers, +custom implementations and score interpretation. -| Argument | Type | Default | Description | -|:---------|:-----|:--------|:------------| -| `normalize` | `str` | `"score"` | The method to normalize the scores. Can be `rank` or `score`. If `rank`, the scores are converted to ranks and then normalized. If `score`, the scores are normalized directly. | -| `reranker` | `Reranker` | `RRF()` | The reranker to use. If not specified, the default reranker is used. | +In synchronous Python, `.rerank(...)` accepts a reranker and a `normalize` +option (`"score"` or `"rank"`). Normalization matters for methods that blend +raw scores; RRF uses rank positions. These are Python query options, not a +shared signature across every SDK. diff --git a/docs/web/search/index.mdx b/docs/web/search/index.mdx index 886751331..9d439b204 100644 --- a/docs/web/search/index.mdx +++ b/docs/web/search/index.mdx @@ -1,88 +1,118 @@ --- title: Search sidebarTitle: Overview -description: "Comprehensive guide to all search capabilities in LanceDB including vector search, full-text search, hybrid search, and more." +description: "Choose vector, full-text or hybrid search and return the records your application needs." icon: "list" +availability: + oss: available + enterprise: available + summary: >- + Vector, full-text and hybrid search are available in OSS and Enterprise. --- -| Feature | Description | -|:---------------|:------------| -| [Vector Search](/search/vector-search/) | Semantic similarity search with multiple distance metrics | -| [Multivector Search](/search/vector-search) | Search using multiple vector embeddings per document | -| [Full-Text Search](/search/full-text-search/) | Keyword-based search with BM25 and pre-filtering | -| [Hybrid Search](/search/hybrid-search/) | Combines vector and full-text search with reranking | -| [Filtering](/search/filtering/) | Filter results based on metadata fields | -| [SQL Queries](/sql) | SQL query capabilities for data exploration and analytics | +import { PyVectorSearch, TsVectorSearchBody, TsVectorSearchTail } from '/snippets/search_examples.mdx'; +import { RsExampleBody as RsVectorSearchBody, RsExampleTail as RsVectorSearchTail } from '/snippets/search_vector_search.mdx'; + +LanceDB supports vector, full-text and hybrid search. Use vector search to find +similar content, full-text search to match words, or hybrid search to combine +both. Filters narrow the results to records that match your criteria. + +## Choose a search method + +| You want to find | Use | Prepare | +|:----------------|:----|:--------| +| Items similar to a query, such as related passages or images | [Vector search](/search/vector-search) | Stored vectors and a compatible query vector | +| Documents represented by several embeddings | [Multivector search](/search/vector-search#multivector) | A multivector column and compatible query vectors | +| Records containing particular words or phrases | [Full-text search](/search/full-text-search) | An FTS index on the text column | +| Relevant meaning alongside keyword matches | [Hybrid search](/search/hybrid-search) | Vectors and an FTS index | +| Rows matching a condition, without similarity ranking | [Filtering](/search/filtering) | A predicate over table columns | + +Vector search can run without an index. An approximate nearest-neighbor index +reduces the work needed for larger tables, with a tradeoff between recall and +latency. Full-text search needs an FTS index before you query it. ## Before you search {#before-you-search} -- Vector search can run without an ANN index as an exhaustive scan. That's useful while prototyping, but build a vector index before relying on low-latency searches over larger tables. -- Full-text and hybrid text search require an FTS index on the text column you query. If a table has multiple FTS indexes, specify the target column. FTS also supports phrase, boolean, boosted, multi-match, and fuzzy query forms when you need more than plain terms. -- Phrase FTS queries require an index created with token positions enabled. -- Multivector search currently uses cosine similarity and accepts either one query vector or a matrix of query vectors; every query vector must match the inner dimension of the multivector column. -- Set an explicit `.limit(...)` for production queries. The default top-k is 10 for search builders, - but spelling it out makes latency and result-count assumptions visible. Query builders also - support controls such as prefilter/postfilter, distance ranges, row-id inclusion, offset - pagination, and Arrow/Pandas/list result materialization. +For vector search, generate stored and query vectors with the same embedding +model and compatible settings. Their dimensions must match. Keep the fields +you want to return, such as document text or an item ID, in the same table. -## Performance {#performance} +For full-text search, choose the text column and build its index. Enterprise +builds indexes in the background, so wait for the index to be ready before +querying it. Phrase queries also need token positions in the index. See +[FTS indexes](/indexing/fts-index). -Three knobs materially affect query latency, memory use, and recall. Be deliberate about each one on every query: + - - - If queries return more data than you need or take longer than expected, the cause is usually projecting more columns than necessary, or letting the result count run unbounded. The best practice is to always pass both `select()` and `limit()`: + - ```python Python icon="python" - table.search(emb).select(["id", "title"]).limit(20) - ``` - - - If a query with `prefilter=False` returns fewer than `limit` results — sometimes zero — the cause is that post-filter applies the predicate after the top-k is selected, so only candidates that already passed the search are filtered. Pre-filter is the default and guarantees every result satisfies the predicate. Switch to `prefilter=False` only when fewer-than-`limit` results are acceptable. See [Filtering](/search/filtering). - - - If recall is lower than expected, the cause is that the search-time knobs limit how many candidates the index considers. Adjust based on your index type: +These examples use three sample records and hand-written vectors, so no +embedding service is needed. Each program creates a temporary local table +and removes it when finished. - - Quantized indexes (PQ, RQ, SQ): `refine_factor` pulls extra candidates and re-scores them on full vectors. - - HNSW-backed indexes: `ef` at search time. Start at `1.5 × k`, raise toward `10 × k` if recall is short. - - IVF candidate breadth: `nprobes` is auto-tuned; override only when a selective pre-filter leaves too few neighbors. - - +- **Python:** Install with `pip install lancedb`, save as `search.py`, then run `python search.py`. +- **TypeScript:** Install with `npm install @lancedb/lancedb`, save as `search.mts`, then run `node search.mts` with Node.js 22.18 or later. +- **Rust:** Download and extract the [complete Rust examples](/static/code/search-rust.zip), then run `cargo run --locked --bin vector_search` from the extracted directory. -For hybrid search, the default `RRFReranker()` combines vector and FTS results into a single ranking via reciprocal rank fusion. See [Hybrid Search](/search/hybrid-search). + + +{PyVectorSearch} + + +{TsVectorSearchBody + "\n" + TsVectorSearchTail} + + +{RsVectorSearchBody + "\n" + RsVectorSearchTail} + + -#### Avoid materializing the whole table {#avoid-materializing-the-whole-table} +This query returns the puppy and dog records, ordered by vector distance. +It searches the three-row table directly, without an ANN index or an embedding +service. For connection options, see [Connect](/basics/connect). -If you need every row in the table (for training, export, or migration), calling `to_pandas()` or `to_arrow()` will run you out of memory on any non-trivial dataset — both materialize the full table at once. The best practice is to iterate via `table.search(...)` or `table.query(...)`, which work the same way in both LanceDB OSS and Enterprise. + -In OSS, you can also stream batches through the underlying Lance dataset directly — useful when you need filter pushdown or fragment-level parallelism via `ds.scanner(...)`: +## Read the results -```python Python icon="python" -ds = table.to_lance() -for batch in ds.to_batches(columns=["id", "text"], batch_size=10000): - process(batch) -``` +Use `select()` to choose the columns your application needs and `limit()` to +set the maximum number of results. The search also returns a ranking value: - -**API disparity between OSS and Enterprise.** +| Search method | Ranking column | Order | +|:--------------|:---------------|:------| +| Vector | `_distance` | Smaller distance first | +| Full-text | `_score` | Higher text relevance first | +| Hybrid | `_relevance_score` | Higher combined relevance first | -LanceDB OSS exposes the `to_pandas()`, `to_arrow()`, and `table.to_lance()` methods for direct Lance dataset access. Enterprise's `RemoteTable` exposes none of these. Only `table.search(...)` and `table.query(...)`. In general, it's always best to go through `search()` and `query()` to keep your code portable across both OSS and Enterprise. - +These values have different meanings. Do not compare a vector distance directly +with a full-text score or treat a reranker score as a probability. -#### Diagnostics {#diagnostics} +## Add filters and reranking {#performance} -To analyze a slow query, inspect what the query engine actually did and the state of the indexes it touched. These two tools surface that information — use them in this order: +[Filters](/search/filtering) restrict which rows qualify, such as documents +from a particular collection. Prefiltering searches within those rows. +Postfiltering removes nonmatching rows from the candidates already retrieved, +so it can leave fewer results than requested. -```python Python icon="python" -print(table.search(emb).where("year > 2000").limit(10).analyze_plan()) -print(table.index_stats("vector_idx")) # num_unindexed_rows should be ~0 -``` +[Reranking](/reranking) reorders the candidates returned by retrieval. Start +with representative queries and expected results, then compare changes to +relevance and latency. A reranker cannot recover a relevant row that retrieval +never returned. -`analyze_plan()` returns the execution plan with per-stage timings, so you can see where the query actually spent its time. `index_stats()` shows how many rows are still unindexed — you want `num_unindexed_rows` to be ~0. In `analyze_plan`, look for: +## Read large result sets in batches {#avoid-materializing-the-whole-table} -| Plan pattern | Fix | -|--------------|-----| -| `LanceScan` with high `bytes_read` / `iops` | Add a missing index, project columns with `select()`, or check that the dataset has been compacted | -| Multiple sequential filters | Reorder filter conditions | +Materializing all rows at once can exhaust application memory. For exports and +training reads, use bounded queries or the batch-reading interfaces described +in [Data loading](/training/data-loading). The SDK buffers Enterprise +query responses before exposing batches. Set a query limit rather than relying +on batch iteration alone to bound memory. -[Explain Analyze](/search/explain) walks through a fully worked before/after example, including how `KNNVectorDistance` and `output_batches` change once indexes are in place. +Direct Lance dataset access through Python's `table.to_lance()` is an OSS +interface. Use the SDK query builders for reads that need to work with both +OSS and Enterprise. + +## Investigate a slow query {#diagnostics} + +Use [query plans](/search/explain) to see the scans, filters and indexes selected +for a query. `explain_plan()` inspects the plan; `analyze_plan()` executes the +query and reports measurements. Compare one change at a time, such as adding +an index, returning fewer columns, or adjusting search parameters. diff --git a/docs/web/search/vector-search.mdx b/docs/web/search/vector-search.mdx index c7324701a..3519df553 100644 --- a/docs/web/search/vector-search.mdx +++ b/docs/web/search/vector-search.mdx @@ -1,8 +1,13 @@ --- title: "Vector Search" sidebarTitle: "Vector Search" -description: "Learn how to run vector search queries in LanceDB. Includes best practices, tips and examples." +description: "Search by vector similarity, choose a distance metric and tune your queries." icon: "arrow-up-right-dots" +availability: + oss: available + enterprise: available + summary: >- + Vector search is available in OSS and Enterprise. --- import { @@ -43,31 +48,68 @@ import { RsBinarySearch, } from '/snippets/search.mdx'; -Vector search is a technique used to search for similar items based on their vector representations, called embeddings. It is also known as similarity search, nearest neighbor search, or approximate nearest neighbor search. +import { PyVectorSearch, TsVectorSearchBody, TsVectorSearchTail } from '/snippets/search_examples.mdx'; +import { RsExampleBody as RsVectorSearchBody, RsExampleTail as RsVectorSearchTail } from '/snippets/search_vector_search.mdx'; -![](/static/assets/images/search/vector-db-basics.png) +Vector search finds similar items by comparing their embeddings: numerical +representations of text, images, audio or other data. An embedding model maps +related items to nearby points in vector space, so you can search by meaning +or similarity instead of matching exact words. -Raw data (e.g. text, images, audio, etc.) is converted into embeddings via an embedding model, which are then stored in a multimodal lakehouse like LanceDB. To perform similarity search at scale, an index is created on the stored embeddings, which can then used to perform fast lookups. +![An embedding model maps data into vectors for similarity search.](/static/assets/images/search/vector-db-basics.png) - -Vector search has **no SQL form**. Run it here and use [SQL](/sql) over the result. - +To run a search, pass a query vector to LanceDB and specify how many matches +to return. Generate the query vector with the same model and settings used for +your stored embeddings. LanceDB ranks the results by distance from the query. -## Supported distance metrics +For small tables, you can search the vectors directly. A [vector index](/indexing/vector-index) +speeds up searches on larger tables by narrowing the candidates to examine. -Distance metrics determine how LanceDB compares vectors to find similar matches. Euclidean or `l2` is the default, and used for general-purpose similarity, `cosine` for unnormalized embeddings, `dot` for normalized embeddings (best performance), or `hamming` for binary vectors. + - -Ensure you always use the same distance metric that your embedding model was trained with. Most modern embedding models use cosine similarity, so `cosine` is often the best choice. However, if your vectors are normalized, you should use `dot` for best performance. - + -The right metric improves both search accuracy and query performance. Currently, LanceDB supports the following metrics: +These examples create a three-row table and find the two closest matches to a +query vector. They use hand-written vectors, so you can try the query without +an embedding model. The temporary table is removed when the program finishes. + +- **Python:** Install with `pip install lancedb`, save as `vector_search.py`, then run `python vector_search.py`. +- **TypeScript:** Install with `npm install @lancedb/lancedb`, save as `vector_search.mts`, then run `node vector_search.mts` with Node.js 22.18 or later. +- **Rust:** Download and extract the [complete Rust examples](/static/code/search-rust.zip), then run `cargo run --locked --bin vector_search` from the extracted directory. + + + +{PyVectorSearch} + + +{TsVectorSearchBody + "\n" + TsVectorSearchTail} + + +{RsVectorSearchBody + "\n" + RsVectorSearchTail} + + + +The first result is the puppy record, whose vector exactly matches the query. +The dog record is next. `_distance` is smaller for closer matches. This example +uses an exhaustive scan; the indexing section explains how to search larger tables. + + + +## Supported distance metrics {#supported-distance-metrics} + +A distance metric determines how LanceDB compares vectors. Choose the metric +recommended by your embedding model. Use the same metric when building an index +and querying it. + +LanceDB supports `l2`, `cosine`, `dot` and `hamming`. An unindexed search uses +`l2` by default; Hamming distance is for binary vectors. The table below shows +the value returned in the `_distance` column. Smaller values rank first. | Distance metric | Mathematical form | Notes | |---|---|---| -| `l2` | $$\|x-y\|_2= \sqrt{\sum_i (x_i-y_i)^2}$$ | The $\ell_2$ norm. Measures Euclidean distance between two points in geometric space. | +| `l2` | $\sum_i (x_i-y_i)^2$ | Squared Euclidean distance returned in `_distance`. | | `cosine` | $1-\frac{x\cdot y}{\|x\|_2\|y\|_2}$ | Measures directional difference between vectors. Insensitive to nonzero scaling of vectors. | -| `dot` | $x\cdot y= \underset{i}{\sum} x_i y_i$ | The standard dot product of real vectors. Sensitive to nonzero scaling of vectors; use on normalized vectors. | +| `dot` | $1-\sum_i x_i y_i$ | Dot-product distance. For unit-normalized vectors, this equals cosine distance. | | `hamming` | $\underset{i}{\sum} \,\: \mathbf{1}_{\, [x_i\neq y_i]}$ | Counts the number of bitwise-differing bits between binary vectors. | For indexed search, supported distance metrics vary by index type: @@ -111,12 +153,10 @@ non-`l2` index, explicitly set the matching metric or upgrade the SDK. -Here you can see the same search but using `cosine` similarity instead of `l2` -distance. If the table has a vector index, make sure it was also built with -`cosine`. The result focuses on vector direction rather than absolute distance, -which works better for unnormalized embeddings. +These examples use cosine distance and return up to ten matches. If the table +has a vector index, build it with the same metric. -Set `.limit(...)` on vector searches you run in applications. You can page through results with `.offset(...)` and include LanceDB's internal row id with `.with_row_id()` / `.withRowId()` when you need a stable handle for follow-up operations. +Set `.limit(...)` on vector searches you run in applications. You can page through results with `.offset(...)` and include LanceDB's internal row id with `.with_row_id()` / `.withRowId()` when an operation requires internal row IDs. Use your own ID column for application identifiers. ## Selecting the vector column @@ -153,13 +193,12 @@ The same field-path syntax works when creating an index on a nested vector colum When several columns share a name across structs (for example, `image.embedding` and `text.embedding`), LanceDB still picks the one whose dimension matches your query vector. If two candidates have the same dimension, you must pass the column name explicitly. -## Vector Search With ANN Index +## Search with a vector index {#vector-search-with-ann-index} -Instead of performing an exhaustive search on the entire database for each and every query, approximate nearest neighbour (ANN) algorithms use an index to narrow down the search space, which significantly reduces query latency. - -The trade-off is that the results are not guaranteed to be the true nearest neighbors of the query, but are usually "good enough" for most use cases. - -Use ANN search for large-scale applications where speed matters more than perfect recall. LanceDB uses approximate nearest neighbor algorithms to deliver fast results without examining every vector in your dataset. +An approximate nearest-neighbor (ANN) index narrows the set of vectors examined +for a query. It can reduce query latency, but may miss neighbors that an +exhaustive search would find. Compare recall and latency on queries that +represent your application before choosing an index and its search settings. When a vector index is used, `_distance` is not always the true distance between full vectors. On quantized ANN indexes, LanceDB may compute `_distance` from the compressed representation for speed. Use `refine_factor` when you want reranking on full vectors. @@ -193,8 +232,8 @@ The table below summarizes the behavior of `_distance` in search results based o -For deeper tuning guidance on indexing and performance estimation, see the [vector indexes](/indexing/vector-index) page, -For tuning `nprobes`, see below. +See [vector indexes](/indexing/vector-index) for index configuration. The +following sections cover query-time settings. ### Tuning `approx_mode` @@ -208,13 +247,13 @@ The supported values are: | `normal` | Use the default balance between query latency and recall. This only has an effect for RQ indexes built with `num_bits > 1`. | | `accurate` | Prefer higher recall, which can increase query latency. | -You can change `approx_mode` per query without rebuilding the index. For RQ indexes built with `num_bits=1`, `normal` uses the same one-bit scoring path as `fast`. If you also set `refine_factor`, LanceDB first uses `approx_mode` while finding candidates, then reranks the selected candidates on full vectors. +Only the Rust SDK sets `approx_mode`; the Python and TypeScript SDKs do not expose it. You can change `approx_mode` per query without rebuilding the index. For RQ indexes built with `num_bits=1`, `normal` uses the same one-bit scoring path as `fast`. If you also set `refine_factor`, LanceDB first uses `approx_mode` while finding candidates, then reranks the selected candidates on full vectors. ### Tuning `nprobes` - `nprobes` controls how many partitions are searched at query time. - `nprobes` improves candidate recall, but does not by itself make `_distance` exact. -- By default, LanceDB automatically tunes `nprobes` to achieve the best performance without noticeably sacrificing accuracy. +- The default probe selection is a starting point. Measure recall and latency for your data before overriding it. - In most cases, leave `nprobes` unset and use the auto-tuned value. - Only tune `nprobes` manually when recall is below your target, or when you need even higher performance for your workload. - If recall is too low, increase `nprobes` gradually, but after a certain threshold, increasing `nprobes` yields only marginal accuracy gains. @@ -223,6 +262,8 @@ You can change `approx_mode` per query without rebuilding the index. For RQ inde For filtered ANN searches, you can also set `minimum_nprobes` and `maximum_nprobes`. LanceDB starts with the minimum and can scan more partitions up to the maximum if the filter leaves too few candidates. Calling `nprobes(n)` fixes both values to `n`, which disables that adaptive behavior. +Whether a deployment honors this range depends on its version; one that does not searches exactly +`minimum_nprobes` partitions. ### Vector Search with Prefiltering @@ -242,15 +283,15 @@ This is the default vector search setting. You can use prefiltering to boost que -This filters out rows where label ≤ 2 before doing vector search, then picks specific columns from the top 5 matches. - -The `.where("label > 2")` applies a filter before vector search, `.select(["text", "keywords", "label"])` chooses specific columns to return, and `.limit(5)` restricts results to the top `5` most similar vectors. - -As a result, you'll see a result with just the data you want from the most similar vectors. +The filter limits which rows can match. Use `select()` to choose the returned +columns and `limit()` to set the maximum number of results. ### Vector Search with Postfiltering -Use postfiltering to prioritize vector similarity by searching the full dataset first, then applying metadata filters to the top results. This approach ensures you get the most similar vectors before filtering, which can be crucial when similarity is more important than metadata constraints. +Postfiltering selects vector-search candidates before applying the metadata +predicate. It can return fewer matches than requested, even when other rows in +the table satisfy the predicate. Use prefiltering when you want the nearest +records within the matching subset. @@ -266,11 +307,8 @@ Use postfiltering to prioritize vector similarity by searching the full dataset -Here you can see how to do vector search first to get the most similar vectors, then filter by label > 1 on those results. - -The `prefilter=False` parameter tells LanceDB to apply the filter after vector search instead of before, `.where("label > 1")` filters the top results by metadata, and `.select()` chooses which columns to include. - -In the end, you receive a query result with the best matches that also meet your metadata requirements. +Only retrieved candidates that satisfy the predicate remain in the result. +Increase the candidate limit or use prefiltering if too few records are returned. [Post-filtering](/search/filtering/#post-filtering-with-vector-search) in LanceDB applies @@ -282,7 +320,7 @@ the filter condition after obtaining the nearest neighbors based on vector simil Use multivector search when your documents contain multiple embeddings and you need sophisticated matching between query and document vector pairs. The late interaction approach finds the most relevant combinations across all available embeddings and provides nuanced similarity scoring. Only `cosine` similarity is supported as the distance metric for multivector search operations. -Every query vector must match the inner dimension of the multivector column; LanceDB rejects mismatched query dimensions rather than guessing how to reshape them. +Every query vector must match the inner dimension of the multivector column. @@ -292,7 +330,7 @@ Every query vector must match the inner dimension of the multivector column; Lan Here you can see how to take 2 query vectors and find the best matching pairs between them and document vectors using late interaction. The `np.random.random(size=(2, 256))` creates a 2×256 array with two random query vectors, `.limit(5)` returns the top 5 best document-query combinations, and `.to_pandas()` provides results in a DataFrame format. -**Read more:** [Multivector search](/search/vector-search) +**Read more:** [Multivector search walkthrough](#multivector) ## Advanced Search Scenarios @@ -350,9 +388,11 @@ The search produces an Arrow table with binary vectors ranked by how many bits d ## Scaling Vector Search -### Batch Search +### Query multiple vectors in a batch {#batch-search} -Use batch search to handle multiple query vectors simultaneously. This gives you significant efficiency gains over individual queries. LanceDB processes all vectors in parallel and organizes results with a `query_index` field that maps each result set back to its originating query. +Batch search accepts multiple query vectors in one request. The `query_index` +column identifies which query produced each result. Measure throughput and +latency with your workload when choosing a batch size. @@ -368,26 +408,16 @@ Use batch search to handle multiple query vectors simultaneously. This gives you -This takes 5 query embeddings and finds the top 5 matches for each one in a single batch operation. +Each input vector has its own nearest-neighbor results. Use `query_index` to +associate each row with the corresponding input vector. -The `load_dataset()` loads embeddings from a Hugging Face dataset, `query_embeds` contains `5` query vectors, and `.search(query_embeds)` processes all queries simultaneously. - -The final query result contains all results, including a `query_index` to tell you which query each result came from. - - -When processing batch queries, the results include a `query_index` field -to explicitly associate each result set with its corresponding query in -the input batch. - - -### Search With Asynchronous Indexing +### Include or skip unindexed rows {#search-with-asynchronous-indexing} To optimize for speed over completeness, enable the `fast_search` flag in your query to skip searching unindexed data. -While vector indexing occurs asynchronously, newly added vectors are immediately -searchable through a fallback brute-force search mechanism. This ensures zero -latency between data insertion and searchability, though it may temporarily -increase query response times. +By default, search includes both indexed vectors and unindexed rows visible to +the query. LanceDB scans the unindexed rows directly. `fast_search` skips that +scan, which can reduce query time but omit recent records. @@ -403,13 +433,9 @@ increase query response times. -Here you can see how to turn on fast search mode to skip unindexed vectors and only look through indexed data for speed. +Use this option only when results from indexed data are sufficient. -The `fast_search=True` parameter tells LanceDB to only search indexed vectors, skipping any recently added data that hasn't been indexed yet. - -You'll obtain a query result with the top `5` matches from indexed vectors, but might miss data that was just added. - -## Brute Force Search +## Search without a vector index {#brute-force-search} ### Search With No Index @@ -433,11 +459,13 @@ Choose brute force search when you need guaranteed 100% recall, typically with s -This carries out a brute force search through every vector in the table to find the 3 closest matches to a random 1536-dimensional query. You'll get back a list of the most similar vectors with exact distances. +These queries return the three nearest matches. Without a vector index, LanceDB +calculates distances directly from the stored vectors. ![](/static/assets/images/search/knn_search.png) -As you can imagine, the brute force approach is not scalable for datasets larger than a few hundred thousand vectors, as the latency of the search grows linearly with the size of the dataset. This is where approximate nearest neighbour (ANN) algorithms come in. +Exhaustive search does more work as the table grows. When query latency becomes +too high, add an ANN index and compare its results with the exhaustive baseline. ### Bypass the Vector Index @@ -461,7 +489,8 @@ This skips the approximate index and checks every single vector for exact, groun The `.bypass_vector_index()` method forces LanceDB to perform an exhaustive search through all vectors instead of using the approximate nearest neighbor index, ensuring exact results but at the cost of slower performance. -The output is a query result with the top 5 exact matches, guaranteeing 100% recall but taking longer to run. +The query returns the nearest five rows within its filters and distance bounds. +Use an exhaustive scan as a baseline when measuring ANN recall. This approach is particularly useful when: - Evaluating ANN index quality @@ -596,7 +625,7 @@ tbl.create_index(metric="cosine", vector_column_name="vector") A brute-force scan over a multivector column has to compare every query vector to every document vector in every row, so the cost grows with both the row count and the number of vectors per row. In LanceDB OSS, the query will just run, so a large unindexed multivector table can stall a process for a long time before returning results. -On LanceDB Cloud, the brute-force KNN safety check applies a stricter row threshold to multivector columns — roughly 10× lower than for single-vector columns. So an unindexed multivector table can be rejected with a "vector search would use brute-force KNN" error well before a comparable single-vector table would. LanceDB Enterprise doesn't have the limit. Build the index before you start hitting it from production traffic, even if the dataset is small enough that you'd skip indexing for a single-vector workload. +On LanceDB Cloud, the brute-force KNN safety check applies a stricter row threshold to multivector columns; roughly 10× lower than for single-vector columns. So an unindexed multivector table can be rejected with a "vector search would use brute-force KNN" error well before a comparable single-vector table would. LanceDB Enterprise doesn't have the limit. Build the index before you start hitting it from production traffic, even if the dataset is small enough that you'd skip indexing for a single-vector workload. #### 6. Query a Single Vector @@ -644,7 +673,7 @@ import lancedb from pylate import models ## 1) Load a late-interaction model via PyLate -## PyLate docs show ColBERT() + encode(..., is_query=...) :contentReference[oaicite:2]{index=2} +## PyLate docs show ColBERT() + encode(..., is_query=...) model = models.ColBERT(model_name_or_path="lightonai/GTE-ModernColBERT-v1") ## You can discover dim from one embedding (avoid guessing) @@ -655,7 +684,7 @@ db = lancedb.connect("./pylate_lancedb") schema = pa.schema([ pa.field("doc_id", pa.string()), pa.field("text", pa.string()), - # multivector: list> :contentReference[oaicite:3]{index=3} + # multivector: list> pa.field("mv", pa.list_(pa.list_(pa.float32(), dim))), ]) @@ -667,7 +696,7 @@ docs = [ ## 3) Encode documents with PyLate (token vectors per doc) doc_texts = [d["text"] for d in docs] -doc_embs = model.encode(doc_texts, is_query=False) # list/array of (T, dim) per doc :contentReference[oaicite:4]{index=4} +doc_embs = model.encode(doc_texts, is_query=False) # list/array of (T, dim) per doc rows = [] for d, emb in zip(docs, doc_embs): @@ -678,14 +707,14 @@ tbl = db.create_table("docs", data=rows, schema=schema, mode="overwrite") ## 4) Build an index + query using a query matrix. ## Multivector brute-force scales with rows × vectors-per-row, so build the index -## at much smaller dataset sizes than you would for single-vector search — and +## at much smaller dataset sizes than you would for single-vector search; and ## always before exposing the table to remote traffic. tbl.create_index(vector_column_name="mv", metric="cosine") query = "Tell me about ramen in Japan" -q_emb = np.asarray(model.encode([query], is_query=True)[0], dtype=np.float32) # (Tq, dim) :contentReference[oaicite:5]{index=5} +q_emb = np.asarray(model.encode([query], is_query=True)[0], dtype=np.float32) # (Tq, dim) -out = tbl.search(q_emb).limit(5).to_pandas() # multivector search accepts a matrix :contentReference[oaicite:6]{index=6} +out = tbl.search(q_emb).limit(5).to_pandas() # multivector search accepts a matrix print(out[["doc_id", "text"]]) ``` diff --git a/docs/web/snippets/connection.mdx b/docs/web/snippets/connection.mdx index c7d8a187d..c145c1fb6 100644 --- a/docs/web/snippets/connection.mdx +++ b/docs/web/snippets/connection.mdx @@ -4,31 +4,37 @@ export const PyConnect = "import lancedb\n\nuri = \"ex_lancedb\"\ndb = lancedb.c export const PyConnectAsync = "import lancedb\n\nuri = \"ex_lancedb\"\nasync_db = await lancedb.connect_async(uri)\n"; -export const PyConnectEnterpriseQuickstart = "uri = \"db://your-database-uri\"\napi_key = \"your-api-key\"\nregion = \"us-east-1\"\nhost_override = \"https://your-enterprise-endpoint.com\"\n\ndb = lancedb.connect(\n uri=uri,\n api_key=api_key,\n region=region,\n host_override=host_override,\n)\n"; +export const PyConnectEnterprisePlaceholders = "import lancedb\n\ndb = lancedb.connect(\n uri=\"db://your-database\",\n api_key=\"your-api-key\",\n region=\"us-east-1\",\n host_override=\"https://your-endpoint\",\n)\n"; + +export const PyConnectEnterpriseQuickstart = "import os\n\nimport lancedb\n\ndb = lancedb.connect(\n uri=os.environ[\"LANCEDB_URI\"], # db://your-database\n api_key=os.environ[\"LANCEDB_API_KEY\"],\n region=os.environ.get(\"LANCEDB_REGION\", \"us-east-1\"),\n host_override=os.environ[\"LANCEDB_HOST_OVERRIDE\"], # https://your-endpoint\n)\n"; export const PyConnectObjectStorage = "import lancedb\n\nuri = \"s3://your-bucket/path\"\n# You can also use \"gs://your-bucket/path\" or \"az://your-container/path\".\ndb = lancedb.connect(uri)\n"; export const PyConnectObjectStorageAsync = "import lancedb\n\nuri = \"s3://your-bucket/path\"\n# You can also use \"gs://your-bucket/path\" or \"az://your-container/path\".\nasync_db = await lancedb.connect_async(uri)\n"; -export const PyNamespaceAdminOps = "import lancedb\n\ndb = lancedb.connect_namespace(\"dir\", {\"root\": \"./local_lancedb\"})\nnamespace = [\"prod\", \"search\"]\n\ndb.create_namespace([\"prod\"])\ndb.create_namespace([\"prod\", \"search\"])\n\nchild_namespaces = db.list_namespaces(namespace_path=[\"prod\"]).namespaces\nprint(f\"Child namespaces under {namespace}: {child_namespaces}\")\n# Child namespaces under ['prod', 'search']: ['search']\n\nmetadata = db.describe_namespace([\"prod\", \"search\"])\nprint(f\"Metadata for namespace {namespace}: {metadata}\")\n# Metadata for namespace ['prod', 'search']: properties=None\n\ndb.drop_namespace([\"prod\", \"search\"], mode=\"skip\")\ndb.drop_namespace([\"prod\"], mode=\"skip\")\n"; +export const PyNamespaceAdminOps = "import lancedb\n\ndb = lancedb.connect_namespace(\"dir\", {\"root\": \"./local_lancedb\"})\nnamespace = [\"prod\", \"search\"]\n\ndb.create_namespace([\"prod\"])\ndb.create_namespace([\"prod\", \"search\"])\n\nchild_namespaces = db.list_namespaces(namespace_path=[\"prod\"]).namespaces\nprint(f\"Child namespaces under ['prod']: {child_namespaces}\")\n# Child namespaces under ['prod']: ['search']\n\nmetadata = db.describe_namespace([\"prod\", \"search\"])\nprint(f\"Metadata for namespace {namespace}: {metadata}\")\n# Metadata for namespace ['prod', 'search']: properties=None\n\ndb.drop_namespace([\"prod\", \"search\"], mode=\"skip\")\ndb.drop_namespace([\"prod\"], mode=\"skip\")\n"; -export const PyNamespaceTableOps = "import lancedb\n\ndb = lancedb.connect_namespace(\"dir\", {\"root\": \"./local_lancedb\"})\n\n# Create namespace tree: prod/search\ndb.create_namespace([\"prod\"], mode=\"exist_ok\")\ndb.create_namespace([\"prod\", \"search\"], mode=\"exist_ok\")\ndb.create_namespace([\"prod\", \"recommendations\"], mode=\"exist_ok\")\n\ndb.create_table(\n \"user\",\n data=[{\"id\": 1, \"vector\": [0.1, 0.2], \"name\": \"alice\"}],\n namespace_path=[\"prod\", \"search\"],\n mode=\"create\", # use \"overwrite\" only if you want to replace existing table\n)\n\ndb.create_table(\n \"user\",\n data=[{\"id\": 2, \"vector\": [0.3, 0.4], \"name\": \"bob\"}],\n namespace_path=[\"prod\", \"recommendations\"],\n mode=\"create\", # use \"overwrite\" only if you want to replace existing table\n)\n\n# Verify\nprint(db.list_namespaces()) # ['prod']\nprint(db.list_namespaces(namespace_path=[\"prod\"])) # ['recommendations', 'search']\nprint(db.list_tables(namespace_path=[\"prod\", \"search\"])) # ['user']\nprint(db.list_tables(namespace_path=[\"prod\", \"recommendations\"])) # ['user']\n"; +export const PyNamespaceTableOps = "import lancedb\n\ndb = lancedb.connect_namespace(\"dir\", {\"root\": \"./local_lancedb\"})\n\n# Create namespace tree: prod/search\ndb.create_namespace([\"prod\"], mode=\"exist_ok\")\ndb.create_namespace([\"prod\", \"search\"], mode=\"exist_ok\")\ndb.create_namespace([\"prod\", \"recommendations\"], mode=\"exist_ok\")\n\ndb.create_table(\n \"user\",\n data=[{\"id\": 1, \"vector\": [0.1, 0.2], \"name\": \"alice\"}],\n namespace_path=[\"prod\", \"search\"],\n mode=\"create\", # use \"overwrite\" only if you want to replace existing table\n)\n\ndb.create_table(\n \"user\",\n data=[{\"id\": 2, \"vector\": [0.3, 0.4], \"name\": \"bob\"}],\n namespace_path=[\"prod\", \"recommendations\"],\n mode=\"create\", # use \"overwrite\" only if you want to replace existing table\n)\n\n# Verify: listings return a response with the names and a page token\nprint(db.list_namespaces().namespaces) # ['prod']\nprint(db.list_namespaces(namespace_path=[\"prod\"]).namespaces)\n# ['recommendations', 'search']\nprint(db.list_tables(namespace_path=[\"prod\", \"search\"]).tables) # ['user']\nprint(db.list_tables(namespace_path=[\"prod\", \"recommendations\"]).tables) # ['user']\n"; -export const TsConnect = "import * as lancedb from \"@lancedb/lancedb\";\n\nasync function connectExample(uri: string) {\n const db = await lancedb.connect(uri);\n return db;\n}\n"; +export const TsConnect = "const db = await lancedb.connect(\"ex_lancedb\");\n"; -export const TsConnectEnterpriseQuickstart = "const uri = \"db://your-database-uri\";\nconst apiKey = \"your-api-key\";\nconst region = \"us-east-1\";\nconst hostOverride = \"https://your-enterprise-endpoint.com\";\n\nconst db = await lancedb.connect(uri, {\n apiKey,\n region,\n hostOverride,\n});\n"; +export const TsConnectEnterprisePlaceholders = "const db = await lancedb.connect(\"db://your-database\", {\n apiKey: \"your-api-key\",\n region: \"us-east-1\",\n hostOverride: \"https://your-endpoint\",\n});\n"; -export const TsConnectObjectStorage = "async function connectObjectStorageExample() {\n const uri = \"s3://your-bucket/path\";\n // You can also use \"gs://your-bucket/path\" or \"az://your-container/path\".\n const db = await lancedb.connect(uri);\n return db;\n}\n"; +export const TsConnectEnterpriseQuickstart = "const setting = (name: string) => {\n const value = process.env[name];\n if (!value) throw new Error(`Set ${name}`);\n return value;\n};\n\nconst db = await lancedb.connect(setting(\"LANCEDB_URI\"), {\n apiKey: setting(\"LANCEDB_API_KEY\"),\n region: process.env.LANCEDB_REGION ?? \"us-east-1\",\n hostOverride: setting(\"LANCEDB_HOST_OVERRIDE\"),\n});\n"; + +export const TsConnectObjectStorage = "const uri = \"s3://your-bucket/path\";\n// You can also use \"gs://your-bucket/path\" or \"az://your-container/path\".\nconst db = await lancedb.connect(uri);\n"; export const TsNamespaceAdminOps = "const db = await lancedb.connectNamespace(\"dir\", { root: \"./local_lancedb\" });\nconst namespace = [\"prod\", \"search\"];\n\nawait db.createNamespace([\"prod\"]);\nawait db.createNamespace([\"prod\", \"search\"]);\n\nconst childNamespaces = (await db.listNamespaces([\"prod\"])).namespaces;\nconsole.log(`Child namespaces under ${JSON.stringify(namespace)}:`, childNamespaces);\n// Child namespaces under [\"prod\",\"search\"]: [ 'search' ]\n\nconst metadata = await db.describeNamespace([\"prod\", \"search\"]);\nconsole.log(`Metadata for namespace ${JSON.stringify(namespace)}:`, metadata);\n\nawait db.dropNamespace([\"prod\", \"search\"], { mode: \"skip\" });\nawait db.dropNamespace([\"prod\"], { mode: \"skip\" });\n"; export const TsNamespaceTableOps = "const db = await lancedb.connectNamespace(\"dir\", { root: \"./local_lancedb\" });\n\n// Create namespace tree: prod/search and prod/recommendations\nawait db.createNamespace([\"prod\"], { mode: \"exist_ok\" });\nawait db.createNamespace([\"prod\", \"search\"], { mode: \"exist_ok\" });\nawait db.createNamespace([\"prod\", \"recommendations\"], { mode: \"exist_ok\" });\n\nawait db.createTable(\n \"user\",\n [{ id: 1, vector: [0.1, 0.2], name: \"alice\" }],\n [\"prod\", \"search\"],\n { mode: \"create\" }, // use \"overwrite\" only if you want to replace existing table\n);\n\nawait db.createTable(\n \"user\",\n [{ id: 2, vector: [0.3, 0.4], name: \"bob\" }],\n [\"prod\", \"recommendations\"],\n { mode: \"create\" },\n);\n\n// Verify\nconsole.log((await db.listNamespaces()).namespaces); // [\"prod\"]\nconsole.log((await db.listNamespaces([\"prod\"])).namespaces); // [\"recommendations\", \"search\"]\nconsole.log(await db.tableNames([\"prod\", \"search\"])); // [\"user\"]\nconsole.log(await db.tableNames([\"prod\", \"recommendations\"])); // [\"user\"]\n"; -export const RsConnect = "async fn connect_example(uri: &str) {\n let db = connect(uri).execute().await.unwrap();\n let _ = db;\n}\n"; +export const RsConnect = "let db = connect(\"ex_lancedb\").execute().await?;\n"; -export const RsConnectEnterpriseQuickstart = "let uri = \"db://your-database-uri\";\nlet api_key = \"your-api-key\";\nlet region = \"us-east-1\";\nlet host_override = \"https://your-enterprise-endpoint.com\";\n"; +export const RsConnectEnterprisePlaceholders = "let db = connect(\"db://your-database\")\n .api_key(\"your-api-key\")\n .region(\"us-east-1\")\n .host_override(\"https://your-endpoint\")\n .execute()\n .await?;\n"; -export const RsConnectObjectStorage = "let uri = \"s3://your-bucket/path\";\n// You can also use \"gs://your-bucket/path\" or \"az://your-container/path\".\n"; +export const RsConnectEnterpriseQuickstart = "let uri = std::env::var(\"LANCEDB_URI\")?; // db://your-database\nlet api_key = std::env::var(\"LANCEDB_API_KEY\")?;\nlet region = std::env::var(\"LANCEDB_REGION\").unwrap_or_else(|_| \"us-east-1\".to_string());\nlet host_override = std::env::var(\"LANCEDB_HOST_OVERRIDE\")?; // https://your-endpoint\n\nlet db = connect(&uri)\n .api_key(&api_key)\n .region(®ion)\n .host_override(&host_override)\n .execute()\n .await?;\n"; + +export const RsConnectObjectStorage = "let uri = \"s3://your-bucket/path\";\n// You can also use \"gs://your-bucket/path\" or \"az://your-container/path\".\nlet db = connect(uri).execute().await?;\n"; export const RsNamespaceAdminOps = "let mut properties = std::collections::HashMap::new();\nproperties.insert(\"root\".to_string(), \"./local_lancedb\".to_string());\nlet db = lancedb::connect_namespace(\"dir\", properties).execute().await?;\nlet namespace = vec![\"prod\".to_string(), \"search\".to_string()];\n\ndb.create_namespace(lance_namespace::models::CreateNamespaceRequest {\n id: Some(vec![\"prod\".to_string()]),\n ..Default::default()\n})\n.await?;\ndb.create_namespace(lance_namespace::models::CreateNamespaceRequest {\n id: Some(namespace.clone()),\n ..Default::default()\n})\n.await?;\n\nlet child_namespaces = db\n .list_namespaces(lance_namespace::models::ListNamespacesRequest {\n id: Some(vec![\"prod\".to_string()]),\n ..Default::default()\n })\n .await?;\nprintln!(\n \"Child namespaces under {:?}: {:?}\",\n namespace, child_namespaces\n);\n// Child namespaces under [\"prod\", \"search\"]: [\"search\"]\n\ndb.drop_namespace(lance_namespace::models::DropNamespaceRequest {\n id: Some(namespace.clone()),\n ..Default::default()\n})\n.await?;\ndb.drop_namespace(lance_namespace::models::DropNamespaceRequest {\n id: Some(vec![\"prod\".to_string()]),\n ..Default::default()\n})\n.await?;\n"; diff --git a/docs/web/snippets/curation_sampling.mdx b/docs/web/snippets/curation_sampling.mdx new file mode 100644 index 000000000..9c3d482b8 --- /dev/null +++ b/docs/web/snippets/curation_sampling.mdx @@ -0,0 +1,4 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const PySampleDataset = "from tempfile import TemporaryDirectory\n\nimport lancedb\nfrom lancedb.permutation import Permutation, permutation_builder\n\nwith TemporaryDirectory(prefix=\"lancedb-sampling-\") as uri:\n db = lancedb.connect(uri)\n table = db.create_table(\"records\", [{\"id\": i} for i in range(10)])\n splits = (\n permutation_builder(table).split_random(counts=[3, 7], seed=42).execute()\n )\n sample = Permutation.from_tables(table, splits, split=0).with_format(\"arrow\")\n selected_ids = []\n for batch in sample.iter(batch_size=3):\n selected_ids.extend(batch.column(\"id\").to_pylist())\n print(f\"Selected {len(selected_ids)} rows: {selected_ids}\")\n"; + diff --git a/docs/web/snippets/embedding.mdx b/docs/web/snippets/embedding.mdx index 8db4472dc..da3388ea7 100644 --- a/docs/web/snippets/embedding.mdx +++ b/docs/web/snippets/embedding.mdx @@ -34,13 +34,13 @@ export const TsRegisterModelFallback = "const registry = getRegistry();\nregistr export const TsRegisterSecret = "const registry = getRegistry();\nregistry.setVar(\"api_key\", \"sk-...\");\n\nconst func = registry.get(\"openai\")!.create({\n apiKey: \"$var:api_key\",\n});\n"; -export const RsCreateEmbeddingFunction = "use std::sync::Arc;\n\nuse lancedb::embeddings::openai::OpenAIEmbeddingFunction;\n\nlet api_key = std::env::var(\"OPENAI_API_KEY\").expect(\"OPENAI_API_KEY is not set\");\nlet embedding = Arc::new(\n OpenAIEmbeddingFunction::new_with_model(api_key, \"text-embedding-3-small\")\n .expect(\"failed to create OpenAI embedding function\"),\n);\n"; +export const RsCreateEmbeddingFunction = "use std::sync::Arc;\n\nuse lancedb::embeddings::openai::OpenAIEmbeddingFunction;\n\nlet api_key = \"your-openai-api-key\".to_string();\nlet embedding = Arc::new(\n OpenAIEmbeddingFunction::new_with_model(api_key, \"text-embedding-3-small\")\n .expect(\"failed to create OpenAI embedding function\"),\n);\n"; export const RsEmbeddingFunction = "use std::{borrow::Cow, sync::Arc};\n\nuse arrow_array::{Array, FixedSizeListArray, Float32Array};\nuse arrow_schema::{DataType, Field, Schema};\nuse lancedb::{\n connect,\n embeddings::{EmbeddingDefinition, EmbeddingFunction},\n Result,\n};\n\n#[derive(Debug, Clone)]\nstruct MyTextEmbedder {\n dim: usize,\n}\n\nimpl EmbeddingFunction for MyTextEmbedder {\n fn name(&self) -> &str {\n \"my-embedder\"\n }\n\n fn source_type(&self) -> Result> {\n Ok(Cow::Owned(DataType::Utf8))\n }\n\n fn dest_type(&self) -> Result> {\n Ok(Cow::Owned(DataType::new_fixed_size_list(\n DataType::Float32,\n self.dim as i32,\n true,\n )))\n }\n\n fn compute_source_embeddings(&self, source: Arc) -> Result> {\n let values = Arc::new(Float32Array::from(vec![1.0f32; source.len() * self.dim]));\n let field = Arc::new(Field::new(\"item\", DataType::Float32, true));\n Ok(Arc::new(FixedSizeListArray::new(\n field,\n self.dim as i32,\n values,\n None,\n )))\n }\n\n fn compute_query_embeddings(&self, _input: Arc) -> Result> {\n unimplemented!()\n }\n}\n\n#[tokio::main]\nasync fn main() -> Result<()> {\n let db = connect(\"./mydb\").execute().await?;\n db.embedding_registry()\n .register(\"my-embedder\", Arc::new(MyTextEmbedder { dim: 3 }))?;\n\n let schema = Arc::new(Schema::new(vec![Field::new(\"text\", DataType::Utf8, false)]));\n db.create_empty_table(\"mytable\", schema)\n .add_embedding(EmbeddingDefinition::new(\n \"text\",\n \"my-embedder\",\n Some(\"vector\"),\n ))?\n .execute()\n .await?;\n\n Ok(())\n}\n"; -export const RsManualQueryEmbeddings = "use std::{iter::once, sync::Arc};\n\nuse arrow_array::{record_batch, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::StreamExt;\nuse lancedb::{\n connect,\n embeddings::{openai::OpenAIEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},\n query::{ExecutableQuery, QueryBase},\n Result,\n};\n\n#[tokio::main]\nasync fn main() -> Result<()> {\n let db = connect(\"./mydb\").execute().await?;\n let api_key = std::env::var(\"OPENAI_API_KEY\").expect(\"OPENAI_API_KEY is not set\");\n let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model(\n api_key,\n \"text-embedding-3-large\",\n )?);\n db.embedding_registry().register(\"openai\", embedding.clone())?;\n\n let schema = Arc::new(Schema::new(vec![Field::new(\"text\", DataType::Utf8, false)]));\n let table = db\n .create_empty_table(\"mytable\", schema)\n .add_embedding(EmbeddingDefinition::new(\"text\", \"openai\", Some(\"vector\")))?\n .execute()\n .await?;\n\n table\n .add(record_batch!((\"text\", Utf8, [\"This is a test.\", \"Another example.\"]))?)\n .execute()\n .await?;\n\n // Manually generate embeddings for the query (Enterprise path)\n let query = Arc::new(StringArray::from_iter_values(once(\"test example\")));\n let query_vector = embedding.compute_query_embeddings(query)?;\n // --8<-- [start:manual_query_search]\n // query_vector is assumed to already be generated by your embedding function\n let mut results = table.vector_search(query_vector)?.limit(5).execute().await?;\n\n while let Some(batch) = results.next().await {\n println!(\"{:?}\", batch?);\n }\n // --8<-- [end:manual_query_search]\n\n Ok(())\n}\n"; +export const RsManualQueryEmbeddings = "use std::{iter::once, sync::Arc};\n\nuse arrow_array::{record_batch, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::StreamExt;\nuse lancedb::{\n connect,\n embeddings::{openai::OpenAIEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},\n query::{ExecutableQuery, QueryBase},\n Result,\n};\n\n#[tokio::main]\nasync fn main() -> Result<()> {\n let db = connect(\"./mydb\").execute().await?;\n let api_key = \"your-openai-api-key\".to_string();\n let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model(\n api_key,\n \"text-embedding-3-large\",\n )?);\n db.embedding_registry().register(\"openai\", embedding.clone())?;\n\n let schema = Arc::new(Schema::new(vec![Field::new(\"text\", DataType::Utf8, false)]));\n let table = db\n .create_empty_table(\"mytable\", schema)\n .add_embedding(EmbeddingDefinition::new(\"text\", \"openai\", Some(\"vector\")))?\n .execute()\n .await?;\n\n table\n .add(record_batch!((\"text\", Utf8, [\"This is a test.\", \"Another example.\"]))?)\n .execute()\n .await?;\n\n // Manually generate embeddings for the query (Enterprise path)\n let query = Arc::new(StringArray::from_iter_values(once(\"test example\")));\n let query_vector = embedding.compute_query_embeddings(query)?;\n // --8<-- [start:manual_query_search]\n // query_vector is assumed to already be generated by your embedding function\n let mut results = table.vector_search(query_vector)?.limit(5).execute().await?;\n\n while let Some(batch) = results.next().await {\n println!(\"{:?}\", batch?);\n }\n // --8<-- [end:manual_query_search]\n\n Ok(())\n}\n"; export const RsManualQuerySearch = "// query_vector is assumed to already be generated by your embedding function\nlet mut results = table.vector_search(query_vector)?.limit(5).execute().await?;\n\nwhile let Some(batch) = results.next().await {\n println!(\"{:?}\", batch?);\n}\n"; -export const RsOpenaiEmbeddings = "use std::{iter::once, sync::Arc};\n\nuse arrow_array::{record_batch, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::StreamExt;\nuse lancedb::{\n connect,\n embeddings::{openai::OpenAIEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},\n query::{ExecutableQuery, QueryBase},\n Result,\n};\n\n#[tokio::main]\nasync fn main() -> Result<()> {\n let db = connect(\"./mydb\").execute().await?;\n let api_key = std::env::var(\"OPENAI_API_KEY\").expect(\"OPENAI_API_KEY is not set\");\n let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model(\n api_key,\n \"text-embedding-3-large\",\n )?);\n\n db.embedding_registry().register(\"openai\", embedding.clone())?;\n\n let schema = Arc::new(Schema::new(vec![Field::new(\"text\", DataType::Utf8, false)]));\n let table = db\n .create_empty_table(\"mytable\", schema)\n .add_embedding(EmbeddingDefinition::new(\"text\", \"openai\", Some(\"vector\")))?\n .execute()\n .await?;\n\n table\n .add(record_batch!((\"text\", Utf8, [\"This is a test.\", \"Another example.\"]))?)\n .execute()\n .await?;\n\n let query = Arc::new(StringArray::from_iter_values(once(\"test example\")));\n let query_vector = embedding.compute_query_embeddings(query)?;\n let mut results = table.vector_search(query_vector)?.limit(5).execute().await?;\n\n while let Some(batch) = results.next().await {\n println!(\"{:?}\", batch?);\n }\n\n Ok(())\n}\n"; +export const RsOpenaiEmbeddings = "use std::{iter::once, sync::Arc};\n\nuse arrow_array::{record_batch, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::StreamExt;\nuse lancedb::{\n connect,\n embeddings::{openai::OpenAIEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},\n query::{ExecutableQuery, QueryBase},\n Result,\n};\n\n#[tokio::main]\nasync fn main() -> Result<()> {\n let db = connect(\"./mydb\").execute().await?;\n let api_key = \"your-openai-api-key\".to_string();\n let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model(\n api_key,\n \"text-embedding-3-large\",\n )?);\n\n db.embedding_registry().register(\"openai\", embedding.clone())?;\n\n let schema = Arc::new(Schema::new(vec![Field::new(\"text\", DataType::Utf8, false)]));\n let table = db\n .create_empty_table(\"mytable\", schema)\n .add_embedding(EmbeddingDefinition::new(\"text\", \"openai\", Some(\"vector\")))?\n .execute()\n .await?;\n\n table\n .add(record_batch!((\"text\", Utf8, [\"This is a test.\", \"Another example.\"]))?)\n .execute()\n .await?;\n\n let query = Arc::new(StringArray::from_iter_values(once(\"test example\")));\n let query_vector = embedding.compute_query_embeddings(query)?;\n let mut results = table.vector_search(query_vector)?.limit(5).execute().await?;\n\n while let Some(batch) = results.next().await {\n println!(\"{:?}\", batch?);\n }\n\n Ok(())\n}\n"; diff --git a/docs/web/snippets/ingestion_delete.mdx b/docs/web/snippets/ingestion_delete.mdx new file mode 100644 index 000000000..f6f6222ae --- /dev/null +++ b/docs/web/snippets/ingestion_delete.mdx @@ -0,0 +1,4 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const RsExample = "use std::sync::Arc;\n\nuse arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::TryStreamExt;\nuse lancedb::query::{ExecutableQuery, QueryBase};\n\nfn make_users_reader(rows: &[(i64, &str)]) -> Box {\n let schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"name\", DataType::Utf8, false),\n ]));\n let batch = RecordBatch::try_new(\n schema.clone(),\n vec![\n Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.0))),\n Arc::new(StringArray::from_iter_values(rows.iter().map(|r| r.1))),\n ],\n )\n .unwrap();\n Box::new(RecordBatchIterator::new(\n vec![Ok(batch)].into_iter(),\n schema,\n ))\n}\n\n#[tokio::main]\nasync fn main() -> Result<(), Box> {\n let directory = tempfile::tempdir()?;\n let db = lancedb::connect(directory.path().to_str().unwrap())\n .execute()\n .await?;\n let table = db\n .create_table(\n \"users\",\n make_users_reader(&[(1, \"Alice\"), (2, \"Bob\"), (3, \"Charlie\")]),\n )\n .execute()\n .await?;\n let predicate = \"id = 3\";\n let selected: Vec = table\n .query()\n .only_if(predicate)\n .execute()\n .await?\n .try_collect()\n .await?;\n println!(\"Selected: {selected:?}\");\n table.delete(predicate).await?;\n let rows: Vec = table.query().execute().await?.try_collect().await?;\n println!(\"Remaining: {rows:?}\");\n assert_eq!(table.count_rows(None).await?, 2);\n assert_eq!(\n table\n .count_rows(Some(\"id = 1 AND name = 'Alice'\".into()))\n .await?,\n 1\n );\n assert_eq!(\n table\n .count_rows(Some(\"id = 2 AND name = 'Bob'\".into()))\n .await?,\n 1\n );\n assert_eq!(table.count_rows(Some(\"id = 3\".into())).await?, 0);\n Ok(())\n}\n"; + diff --git a/docs/web/snippets/ingestion_examples.mdx b/docs/web/snippets/ingestion_examples.mdx new file mode 100644 index 000000000..0268a906a --- /dev/null +++ b/docs/web/snippets/ingestion_examples.mdx @@ -0,0 +1,66 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const PyAddFromDataset = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n from pathlib import Path\n\n import pyarrow as pa\n import pyarrow.parquet as pq\n\n data_path = str(Path(uri) / \"input.parquet\")\n pq.write_table(pa.table({\"id\": [1, 2], \"name\": [\"Alice\", \"Bob\"]}), data_path)\n import pyarrow.dataset as ds\n\n dataset = ds.dataset(data_path, format=\"parquet\")\n table = db.create_table(\"my_table\", schema=dataset.schema, mode=\"overwrite\")\n table.add(dataset)\n\n print(\"Rows:\", table.count_rows())\n"; + +export const PyAddProgress = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\ndef report_progress(progress):\n print(f\"Rows written: {progress['output_rows']}\")\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n table = db.create_table(\"users\", [{\"id\": 1, \"name\": \"Alice\"}])\n table.add(\n [{\"id\": 2, \"name\": \"Bob\"}, {\"id\": 3, \"name\": \"Charlie\"}],\n progress=report_progress,\n )\n rows = sorted(table.search().to_list(), key=lambda row: row[\"id\"])\n print(\"After insertion:\", rows)\n"; + +export const PyAppendRows = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n table = db.create_table(\"users\", [{\"id\": 1, \"name\": \"Alice\"}])\n table.add([{\"id\": 2, \"name\": \"Bob\"}, {\"id\": 3, \"name\": \"Charlie\"}])\n rows = sorted(table.search().to_list(), key=lambda row: row[\"id\"])\n print(rows)\n"; + +export const PyCreateEmptyTable = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import pyarrow as pa\n\n schema = pa.schema(\n [\n pa.field(\"vector\", pa.list_(pa.float32(), 2)),\n pa.field(\"item\", pa.string()),\n pa.field(\"price\", pa.float32()),\n ]\n )\n tbl = db.create_table(\"test_empty_table\", schema=schema, mode=\"overwrite\")\n\n print(\"Rows:\", tbl.count_rows())\n"; + +export const PyCreateEmptyTablePydantic = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n from lancedb.pydantic import LanceModel, Vector\n\n class Item(LanceModel):\n vector: Vector(2)\n item: str\n price: float\n\n tbl = db.create_table(\n \"test_empty_table_new\", schema=Item.to_arrow_schema(), mode=\"overwrite\"\n )\n\n print(\"Rows:\", tbl.count_rows())\n"; + +export const PyCreateTableConflictHandling = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n data = [{\"vector\": [1.1, 1.2], \"lat\": 45.5, \"long\": -122.7}]\n db.create_table(\"conflict_table\", data)\n # Idempotent open: reuse the existing table if it exists.\n # The provided data is ignored; the schema is validated against the\n # existing table and a mismatch raises an error.\n tbl = db.create_table(\"conflict_table\", data, exist_ok=True)\n\n # Overwrite: replace the contents with the provided data.\n tbl = db.create_table(\"conflict_table\", data, mode=\"overwrite\")\n print(\"Rows:\", tbl.count_rows())\n"; + +export const PyCreateTableCustomSchema = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import pyarrow as pa\n\n custom_schema = pa.schema(\n [\n pa.field(\"vector\", pa.list_(pa.float32(), 4)),\n pa.field(\"lat\", pa.float32()),\n pa.field(\"long\", pa.float32()),\n ]\n )\n\n data = [\n {\"vector\": [1.1, 1.2, 1.3, 1.4], \"lat\": 45.5, \"long\": -122.7},\n {\"vector\": [0.2, 1.8, 0.4, 3.6], \"lat\": 40.1, \"long\": -74.1},\n ]\n tbl = db.create_table(\n \"my_table_custom_schema\", data, schema=custom_schema, mode=\"overwrite\"\n )\n\n print(\"Rows:\", tbl.count_rows())\n"; + +export const PyCreateTableFromArrow = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import numpy as np\n import pyarrow as pa\n\n dim = 16\n total = 2\n schema = pa.schema(\n [\n pa.field(\"vector\", pa.list_(pa.float16(), dim)),\n pa.field(\"text\", pa.string()),\n ]\n )\n data = pa.Table.from_arrays(\n [\n pa.array(\n [np.random.randn(dim).astype(np.float16) for _ in range(total)],\n pa.list_(pa.float16(), dim),\n ),\n pa.array([\"foo\", \"bar\"]),\n ],\n [\"vector\", \"text\"],\n )\n tbl = db.create_table(\"f16_tbl\", data, schema=schema, mode=\"overwrite\")\n\n print(\"Rows:\", tbl.count_rows())\n"; + +export const PyCreateTableFromDicts = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n data = [\n {\"vector\": [1.1, 1.2], \"lat\": 45.5, \"long\": -122.7},\n {\"vector\": [0.2, 1.8], \"lat\": 40.1, \"long\": -74.1},\n ]\n db.create_table(\"test_table\", data, mode=\"overwrite\")\n tbl = db[\"test_table\"]\n tbl.head()\n\n print(\"Rows:\", tbl.count_rows())\n"; + +export const PyCreateTableFromIterator = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import pyarrow as pa\n\n schema = pa.schema(\n [\n pa.field(\"vector\", pa.list_(pa.float32(), 4)),\n pa.field(\"item\", pa.utf8()),\n pa.field(\"price\", pa.float32()),\n ]\n )\n\n def make_batches():\n for i in range(5):\n yield pa.RecordBatch.from_arrays(\n [\n pa.array(\n [[3.1, 4.1, 5.1, 6.1], [5.9, 26.5, 4.7, 32.8]],\n pa.list_(pa.float32(), 4),\n ),\n pa.array([\"foo\", \"bar\"]),\n pa.array([10.0, 20.0]),\n ],\n [\"vector\", \"item\", \"price\"],\n )\n\n table = db.create_table(\"batched_table\", schema=schema)\n table.add(make_batches())\n\n print(\"Rows:\", table.count_rows())\n"; + +export const PyCreateTableFromPandas = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import pandas as pd\n\n data = pd.DataFrame(\n {\n \"vector\": [[1.1, 1.2, 1.3, 1.4], [0.2, 1.8, 0.4, 3.6]],\n \"lat\": [45.5, 40.1],\n \"long\": [-122.7, -74.1],\n }\n )\n db.create_table(\"my_table_pandas\", data, mode=\"overwrite\")\n db[\"my_table_pandas\"].head()\n"; + +export const PyCreateTableFromPolars = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import polars as pl\n\n data = pl.DataFrame(\n {\n \"vector\": [[3.1, 4.1], [5.9, 26.5]],\n \"item\": [\"foo\", \"bar\"],\n \"price\": [10.0, 20.0],\n }\n )\n tbl = db.create_table(\"my_table_pl\", data, mode=\"overwrite\")\n\n print(\"Rows:\", tbl.count_rows())\n"; + +export const PyCreateTableFromPydantic = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n from lancedb.pydantic import LanceModel, Vector\n\n class Content(LanceModel):\n movie_id: int\n vector: Vector(128)\n genres: str\n title: str\n imdb_id: int\n\n @property\n def imdb_url(self) -> str:\n return f\"https://www.imdb.com/title/tt{self.imdb_id}\"\n\n tbl = db.create_table(\"movielens_small\", schema=Content, mode=\"overwrite\")\n\n print(\"Rows:\", tbl.count_rows())\n"; + +export const PyCreateTableNestedSchema = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n from lancedb.pydantic import LanceModel, Vector\n from pydantic import BaseModel\n\n class Document(BaseModel):\n content: str\n source: str\n\n class NestedSchema(LanceModel):\n id: str\n vector: Vector(1536)\n document: Document\n\n tbl = db.create_table(\"nested_table\", schema=NestedSchema, mode=\"overwrite\")\n\n print(\"Rows:\", tbl.count_rows())\n"; + +export const PyDeleteRows = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n table = db.create_table(\n \"users\",\n [\n {\"id\": 1, \"name\": \"Alice\"},\n {\"id\": 2, \"name\": \"Bob\"},\n {\"id\": 3, \"name\": \"Charlie\"},\n ],\n )\n predicate = \"id = 3\"\n print(\"Selected:\", table.search().where(predicate).to_list())\n table.delete(predicate)\n rows = sorted(table.search().to_list(), key=lambda row: row[\"id\"])\n print(\"Remaining:\", rows)\n"; + +export const PyDropTable = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n # Create a table first\n data = [{\"vector\": [1.1, 1.2], \"lat\": 45.5}]\n db.create_table(\"my_table\", data, mode=\"overwrite\")\n\n # Drop the table\n db.drop_table(\"my_table\")\n"; + +export const PyInsertIfNotExists = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import pyarrow as pa\n\n table = db.create_table(\n \"users_example\",\n data=pa.table(\n {\n \"id\": [1, 2],\n \"name\": [\"Alice\", \"Bob\"],\n \"login_count\": [10, 20],\n }\n ),\n mode=\"overwrite\",\n )\n\n incoming_users = pa.table(\n {\n \"id\": [2, 3],\n \"name\": [\"Bobby\", \"Charlie\"],\n \"login_count\": [21, 5],\n }\n )\n\n (table.merge_insert(\"id\").when_not_matched_insert_all().execute(incoming_users))\n\n print(\"Rows:\", table.count_rows())\n"; + +export const PyMergeDeleteMissingBySource = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import pyarrow as pa\n\n table = db.create_table(\n \"users_example\",\n data=pa.table(\n {\n \"id\": [1, 2, 3],\n \"name\": [\"Alice\", \"Bob\", \"Charlie\"],\n \"login_count\": [10, 20, 5],\n }\n ),\n mode=\"overwrite\",\n )\n\n incoming_users = pa.table(\n {\n \"id\": [2, 3],\n \"name\": [\"Bobby\", \"Charlie\"],\n \"login_count\": [21, 5],\n }\n )\n\n (\n table.merge_insert(\"id\")\n .when_matched_update_all()\n .when_not_matched_insert_all()\n .when_not_matched_by_source_delete()\n .execute(incoming_users)\n )\n\n print(\"Rows:\", table.count_rows())\n"; + +export const PyMergeMatchedUpdateOnly = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import pyarrow as pa\n\n table = db.create_table(\n \"users_example\",\n data=pa.table(\n {\n \"id\": [1, 2],\n \"name\": [\"Alice\", \"Bob\"],\n \"login_count\": [10, 20],\n }\n ),\n mode=\"overwrite\",\n )\n\n incoming_users = pa.table(\n {\n \"id\": [2, 3],\n \"name\": [\"Bobby\", \"Charlie\"],\n \"login_count\": [21, 5],\n }\n )\n\n (table.merge_insert(\"id\").when_matched_update_all().execute(incoming_users))\n\n print(\"Rows:\", table.count_rows())\n"; + +export const PyMergePartialColumns = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import pyarrow as pa\n\n table = db.create_table(\n \"users_example\",\n data=pa.table(\n {\n \"id\": [1, 2],\n \"name\": [\"Alice\", \"Bob\"],\n \"login_count\": [10, 20],\n }\n ),\n mode=\"overwrite\",\n )\n\n incoming_users = pa.table(\n {\n \"id\": [2, 3],\n \"name\": [\"Bobby\", \"Charlie\"],\n }\n )\n\n (\n table.merge_insert(\"id\")\n .when_matched_update_all()\n .when_not_matched_insert_all()\n .execute(incoming_users)\n )\n\n print(\"Rows:\", table.count_rows())\n"; + +export const PyMergeUpdateInsert = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import pyarrow as pa\n\n table = db.create_table(\n \"users_example\",\n data=pa.table(\n {\n \"id\": [1, 2],\n \"name\": [\"Alice\", \"Bob\"],\n \"login_count\": [10, 20],\n }\n ),\n mode=\"overwrite\",\n )\n\n incoming_users = pa.table(\n {\n \"id\": [2, 3],\n \"name\": [\"Bobby\", \"Charlie\"],\n \"login_count\": [21, 5],\n }\n )\n\n (\n table.merge_insert(\"id\")\n .when_matched_update_all()\n .when_not_matched_insert_all()\n .execute(incoming_users)\n )\n\n print(\"Rows:\", table.count_rows())\n"; + +export const PyOpenExistingTable = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n # Create a table first\n data = [{\"vector\": [1.1, 1.2], \"lat\": 45.5, \"long\": -122.7}]\n db.create_table(\"test_table\", data, mode=\"overwrite\")\n\n # List table names\n print(db.list_tables().tables)\n\n # Open existing table\n tbl = db.open_table(\"test_table\")\n\n print(\"Rows:\", tbl.count_rows())\n"; + +export const PyUpdateOperation = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import pyarrow as pa\n\n table = db.create_table(\n \"users_example\",\n data=pa.table(\n {\n \"id\": [1, 2],\n \"name\": [\"Alice\", \"Bob\"],\n \"login_count\": [10, 20],\n }\n ),\n mode=\"overwrite\",\n )\n table.update(where=\"id = 2\", values={\"name\": \"Bobby\"})\n\n print(\"Rows:\", table.count_rows())\n"; + +export const PyUpdateUsingSql = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n import pyarrow as pa\n\n table = db.create_table(\n \"users_example\",\n data=pa.table(\n {\n \"id\": [1, 2],\n \"name\": [\"Alice\", \"Bob\"],\n \"login_count\": [10, 20],\n }\n ),\n mode=\"overwrite\",\n )\n table.update(where=\"id = 2\", values_sql={\"login_count\": \"login_count + 1\"})\n\n print(\"Rows:\", table.count_rows())\n"; + +export const PyWriteWorkflow = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-ingestion-\") as uri:\n db = lancedb.connect(uri)\n table = db.create_table(\n \"users\",\n [\n {\"id\": 1, \"name\": \"Alice\", \"login_count\": 10},\n {\"id\": 2, \"name\": \"Bob\", \"login_count\": 20},\n ],\n )\n table.add([{\"id\": 3, \"name\": \"Charlie\", \"login_count\": 5}])\n table.update(where=\"id = 2\", values={\"name\": \"Bobby\"})\n table.update(where=\"id = 2\", values_sql={\"login_count\": \"login_count + 1\"})\n (\n table.merge_insert(\"id\")\n .when_matched_update_all()\n .when_not_matched_insert_all()\n .execute(\n [\n {\"id\": 3, \"name\": \"Charlie\", \"login_count\": 6},\n {\"id\": 4, \"name\": \"Dana\", \"login_count\": 1},\n ]\n )\n )\n print(\"Rows selected for deletion:\", table.search().where(\"id = 4\").to_list())\n table.delete(\"id = 4\")\n rows = sorted(table.search().to_list(), key=lambda row: row[\"id\"])\n print(rows)\n"; + +export const TsAddProgressBody = "const lancedb = await import(\"@lancedb/lancedb\");\nconst { mkdtemp, rm } = await import(\"node:fs/promises\");\nconst { tmpdir } = await import(\"node:os\");\nconst { join } = await import(\"node:path\");\nconst uri = await mkdtemp(join(tmpdir(), \"lancedb-ingestion-\"));\ntry {\n const db = await lancedb.connect(uri);\n const table = await db.createTable(\"users\", [{ id: 1, name: \"Alice\" }]);\n await table.add(\n [\n { id: 2, name: \"Bob\" },\n { id: 3, name: \"Charlie\" },\n ],\n {\n progress: (progress) =>\n console.log(`Rows written: ${progress.outputRows}`),\n },\n );\n const rows = (await table.query().toArray()).sort((a, b) => a.id - b.id);\n console.log(\"After insertion:\", rows);\n"; + +export const TsAddProgressTail = "} finally {\n await rm(uri, { recursive: true, force: true });\n}\n"; + +export const TsDeleteRowsBody = "const lancedb = await import(\"@lancedb/lancedb\");\nconst { mkdtemp, rm } = await import(\"node:fs/promises\");\nconst { tmpdir } = await import(\"node:os\");\nconst { join } = await import(\"node:path\");\nconst uri = await mkdtemp(join(tmpdir(), \"lancedb-ingestion-\"));\ntry {\n const db = await lancedb.connect(uri);\n const table = await db.createTable(\"users\", [\n { id: 1, name: \"Alice\" },\n { id: 2, name: \"Bob\" },\n { id: 3, name: \"Charlie\" },\n ]);\n const predicate = \"id = 3\";\n console.log(\"Selected:\", await table.query().where(predicate).toArray());\n await table.delete(predicate);\n const rows = (await table.query().toArray()).sort((a, b) => a.id - b.id);\n console.log(\"Remaining:\", rows);\n"; + +export const TsDeleteRowsTail = "} finally {\n await rm(uri, { recursive: true, force: true });\n}\n"; + +export const TsWriteWorkflowBody = "const lancedb = await import(\"@lancedb/lancedb\");\nconst { mkdtemp, rm } = await import(\"node:fs/promises\");\nconst { tmpdir } = await import(\"node:os\");\nconst { join } = await import(\"node:path\");\n\nconst uri = await mkdtemp(join(tmpdir(), \"lancedb-ingestion-\"));\ntry {\n const db = await lancedb.connect(uri);\n const table = await db.createTable(\"users\", [\n { id: 1, name: \"Alice\", login_count: 10 },\n { id: 2, name: \"Bob\", login_count: 20 },\n ]);\n await table.add([{ id: 3, name: \"Charlie\", login_count: 5 }]);\n await table.update({ where: \"id = 2\", values: { name: \"Bobby\" } });\n await table.update({\n where: \"id = 2\",\n valuesSql: { login_count: \"login_count + 1\" },\n });\n await table\n .mergeInsert(\"id\")\n .whenMatchedUpdateAll()\n .whenNotMatchedInsertAll()\n .execute([\n { id: 3, name: \"Charlie\", login_count: 6 },\n { id: 4, name: \"Dana\", login_count: 1 },\n ]);\n console.log(\n \"Rows selected for deletion:\",\n await table.query().where(\"id = 4\").toArray(),\n );\n await table.delete(\"id = 4\");\n const rows = (await table.query().toArray()).sort((a, b) => a.id - b.id);\n console.log(rows);\n"; + +export const TsWriteWorkflowTail = "} finally {\n await rm(uri, { recursive: true, force: true });\n}\n"; + +export const RsWriteWorkflow = "use std::sync::Arc;\n\nuse arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::TryStreamExt;\nuse lancedb::query::{ExecutableQuery, QueryBase};\n\nfn make_users_reader(rows: &[(i64, &str, i64)]) -> Box {\n let schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"name\", DataType::Utf8, false),\n Field::new(\"login_count\", DataType::Int64, false),\n ]));\n let batch = RecordBatch::try_new(\n schema.clone(),\n vec![\n Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.0))),\n Arc::new(StringArray::from_iter_values(rows.iter().map(|r| r.1))),\n Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.2))),\n ],\n )\n .unwrap();\n Box::new(RecordBatchIterator::new(\n vec![Ok(batch)].into_iter(),\n schema,\n ))\n}\n\n#[tokio::main]\nasync fn main() -> Result<(), Box> {\n let directory = tempfile::tempdir()?;\n let db = lancedb::connect(directory.path().to_str().unwrap())\n .execute()\n .await?;\n let table = db\n .create_table(\n \"users\",\n make_users_reader(&[(1, \"Alice\", 10), (2, \"Bob\", 20)]),\n )\n .execute()\n .await?;\n table\n .add(make_users_reader(&[(3, \"Charlie\", 5)]))\n .execute()\n .await?;\n table\n .update()\n .only_if(\"id = 2\")\n .column(\"name\", \"'Bobby'\")\n .execute()\n .await?;\n table\n .update()\n .only_if(\"id = 2\")\n .column(\"login_count\", \"login_count + 1\")\n .execute()\n .await?;\n let mut merge = table.merge_insert(&[\"id\"]);\n merge\n .when_matched_update_all(None)\n .when_not_matched_insert_all();\n merge\n .execute(make_users_reader(&[(3, \"Charlie\", 6), (4, \"Dana\", 1)]))\n .await?;\n let selected: Vec = table\n .query()\n .only_if(\"id = 4\")\n .execute()\n .await?\n .try_collect()\n .await?;\n println!(\"Rows selected for deletion: {selected:?}\");\n table.delete(\"id = 4\").await?;\n let rows: Vec = table.query().execute().await?.try_collect().await?;\n println!(\"Remaining rows: {rows:?}\");\n assert_eq!(table.count_rows(None).await?, 3);\n assert_eq!(\n table\n .count_rows(Some(\n \"id = 2 AND name = 'Bobby' AND login_count = 21\".into()\n ))\n .await?,\n 1\n );\n assert_eq!(\n table\n .count_rows(Some(\"id = 3 AND login_count = 6\".into()))\n .await?,\n 1\n );\n Ok(())\n}\n"; + diff --git a/docs/web/snippets/ingestion_progress.mdx b/docs/web/snippets/ingestion_progress.mdx new file mode 100644 index 000000000..d910468f6 --- /dev/null +++ b/docs/web/snippets/ingestion_progress.mdx @@ -0,0 +1,4 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const RsExample = "use std::sync::Arc;\n\nuse arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::TryStreamExt;\nuse lancedb::query::ExecutableQuery;\n\nfn make_users_reader(rows: &[(i64, &str)]) -> Box {\n let schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"name\", DataType::Utf8, false),\n ]));\n let batch = RecordBatch::try_new(\n schema.clone(),\n vec![\n Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.0))),\n Arc::new(StringArray::from_iter_values(rows.iter().map(|r| r.1))),\n ],\n )\n .unwrap();\n Box::new(RecordBatchIterator::new(\n vec![Ok(batch)].into_iter(),\n schema,\n ))\n}\n\n#[tokio::main]\nasync fn main() -> Result<(), Box> {\n let directory = tempfile::tempdir()?;\n let db = lancedb::connect(directory.path().to_str().unwrap())\n .execute()\n .await?;\n let table = db\n .create_table(\"users\", make_users_reader(&[(1, \"Alice\")]))\n .execute()\n .await?;\n table\n .add(make_users_reader(&[(2, \"Bob\"), (3, \"Charlie\")]))\n .progress(|progress| println!(\"Rows written: {}\", progress.output_rows()))\n .execute()\n .await?;\n let rows: Vec = table.query().execute().await?.try_collect().await?;\n println!(\"After insertion: {rows:?}\");\n assert_eq!(table.count_rows(None).await?, 3);\n assert_eq!(\n table\n .count_rows(Some(\"id = 1 AND name = 'Alice'\".into()))\n .await?,\n 1\n );\n assert_eq!(\n table\n .count_rows(Some(\"id = 2 AND name = 'Bob'\".into()))\n .await?,\n 1\n );\n assert_eq!(table.count_rows(Some(\"id = 3\".into())).await?, 1);\n Ok(())\n}\n"; + diff --git a/docs/web/snippets/integrations.mdx b/docs/web/snippets/integrations.mdx index 22bbde338..67bb654cf 100644 --- a/docs/web/snippets/integrations.mdx +++ b/docs/web/snippets/integrations.mdx @@ -88,7 +88,7 @@ export const PyFrameworksLlamaindexHybridSearch = "from lancedb.rerankers import export const PyFrameworksLlamaindexQuickStart = "import logging\nimport sys\nimport textwrap\n\nimport openai\n\n# Uncomment to see debug logs\n# logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)\n# logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))\nfrom llama_index.core import (\n Document,\n SimpleDirectoryReader,\n StorageContext,\n VectorStoreIndex,\n)\nfrom llama_index.vector_stores.lancedb import LanceDBVectorStore\n\nopenai.api_key = \"sk-...\"\n\ndocuments = SimpleDirectoryReader(\"./data/your-data-dir/\").load_data()\nprint(\"Document ID:\", documents[0].doc_id, \"Document Hash:\", documents[0].hash)\n\n## For LanceDB Enterprise :\n# vector_store = LanceDBVectorStore(\n# uri=\"db://db_name\", # your remote DB URI\n# api_key=\"sk_..\", # lancedb enterprise api key\n# region=\"your-region\" # the region you configured\n# host_override=\"https://your-host.com\" # if you have a custom host, otherwise omit this\n# )\n\nvector_store = LanceDBVectorStore(\n uri=\"./lancedb\", mode=\"overwrite\", query_type=\"vector\"\n)\nstorage_context = StorageContext.from_defaults(vector_store=vector_store)\n\nindex = VectorStoreIndex.from_documents(documents, storage_context=storage_context)\nlance_filter = \"metadata.file_name = 'paul_graham_essay.txt' \"\nretriever = index.as_retriever(vector_store_kwargs={\"where\": lance_filter})\nresponse = retriever.retrieve(\"What did the author do growing up?\")\n"; -export const PyFrameworksPydanticBaseExample = "table = db.create_table(\"docs\", schema=LanceDocs, mode=\"overwrite\")\ntable.add(\n [\n {\"text\": \"hello world\", \"vector\": [1.0, 0.0]},\n {\"text\": \"goodbye world\", \"vector\": [0.0, 1.0]},\n ]\n)\nresults = table.search(\"hello world\").limit(1).to_pydantic(LanceDocs)\nprint(results[0].text)\n"; +export const PyFrameworksPydanticBaseExample = "table = db.create_table(\"docs\", schema=LanceDocs, mode=\"overwrite\")\ntable.add(\n [\n {\"text\": \"hello world\", \"vector\": [1.0, 0.0]},\n {\"text\": \"goodbye world\", \"vector\": [0.0, 1.0]},\n ]\n)\nresults = table.search([1.0, 0.0]).limit(1).to_pydantic(LanceDocs)\nprint(results[0].text)\n# hello world\n"; export const PyFrameworksPydanticBaseModel = "class LanceDocs(LanceModel):\n text: str\n vector: Vector(2)\n"; @@ -120,7 +120,7 @@ export const PyPlatformsDuckdbMeanPrice = "duckdb.query(\"SELECT mean(price) FRO export const PyPlatformsDuckdbQueryTable = "import duckdb\n\narrow_table = table.to_lance()\n\nduckdb.query(\"SELECT * FROM arrow_table\")\n"; -export const PyPlatformsPandasAsyncExample = "async def run_pandas_async_example() -> None:\n async_db = await lancedb.connect_async(\n str(Path(tempfile.mkdtemp()) / \"pandas-async\")\n )\n async_df = pd.DataFrame(\n [\n {\"id\": \"10\", \"text\": \"sage\", \"vector\": [0.6, 0.4, 0.8]},\n {\"id\": \"11\", \"text\": \"bard\", \"vector\": [0.2, 0.7, 0.3]},\n ]\n )\n async_table = await async_db.create_table(\n \"creatures_async\", data=async_df, mode=\"overwrite\"\n )\n async_results = await (\n async_table.search([0.6, 0.4, 0.8])\n .select([\"text\", \"_distance\"])\n .limit(1)\n .to_pandas()\n )\n print(async_results)\n\nasyncio.run(run_pandas_async_example())\n"; +export const PyPlatformsPandasAsyncExample = "async def run_pandas_async_example() -> pd.DataFrame:\n async_db = await lancedb.connect_async(\n str(Path(tempfile.mkdtemp()) / \"pandas-async\")\n )\n async_df = pd.DataFrame(\n [\n {\"id\": \"10\", \"text\": \"sage\", \"vector\": [0.6, 0.4, 0.8]},\n {\"id\": \"11\", \"text\": \"bard\", \"vector\": [0.2, 0.7, 0.3]},\n ]\n )\n async_table = await async_db.create_table(\n \"creatures_async\", data=async_df, mode=\"overwrite\"\n )\n # An async search returns its query builder from an await\n query = await async_table.search([0.6, 0.4, 0.8])\n async_results = await query.select([\"text\", \"_distance\"]).limit(1).to_pandas()\n print(async_results)\n return async_results\n\nasync_results = asyncio.run(run_pandas_async_example())\n"; export const PyPlatformsPandasCreateTable = "pandas_df = pd.DataFrame(\n [\n {\"id\": \"1\", \"text\": \"dragon\", \"vector\": [0.9, 0.1, 0.3]},\n {\"id\": \"2\", \"text\": \"griffin\", \"vector\": [0.4, 0.5, 0.2]},\n {\"id\": \"3\", \"text\": \"phoenix\", \"vector\": [0.7, 0.3, 0.6]},\n ]\n)\npandas_db = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"pandas-demo\"))\npandas_table = pandas_db.create_table(\"creatures\", data=pandas_df, mode=\"overwrite\")\n"; @@ -152,23 +152,23 @@ export const PyPlatformsVoxel51LoadDataset = "import fiftyone as fo\nimport fift export const PyPlatformsVoxel51SortBySimilarity = "# Step 4: Query your data\nquery = dataset.first().id # query by sample ID\nview = dataset.sort_by_similarity(\n query,\n brain_key=\"lancedb_index\",\n k=10, # limit to 10 most similar samples\n)\n"; -export const PyRerankingAnswerdotaiUsage = "import lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import AnswerdotaiRerankers\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = AnswerdotaiRerankers()\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; +export const PyRerankingAnswerdotaiUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import AnswerdotaiRerankers\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"answerdotai-rerank\"))\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema)\ntbl.add(data)\nreranker = AnswerdotaiRerankers()\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Build the full-text index before either FTS or hybrid search.\ntbl.create_fts_index(\"text\", replace=True)\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; export const PyRerankingCohereUsage = "import os\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import CohereReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = CohereReranker(api_key=os.environ[\"COHERE_API_KEY\"])\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; -export const PyRerankingColbertUsage = "import lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import ColbertReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = ColbertReranker()\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; +export const PyRerankingColbertUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import ColbertReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"colbert-rerank\"))\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema)\ntbl.add(data)\nreranker = ColbertReranker()\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Build the full-text index before either FTS or hybrid search.\ntbl.create_fts_index(\"text\", replace=True)\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; -export const PyRerankingCrossEncoderUsage = "import lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import CrossEncoderReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = CrossEncoderReranker()\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; +export const PyRerankingCrossEncoderUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import CrossEncoderReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"cross-encoder-rerank\"))\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema)\ntbl.add(data)\nreranker = CrossEncoderReranker()\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Build the full-text index before either FTS or hybrid search.\ntbl.create_fts_index(\"text\", replace=True)\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; export const PyRerankingJinaUsage = "import os\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import JinaReranker\n\nembedder = get_registry().get(\"jina\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = JinaReranker(api_key=os.environ[\"JINA_API_KEY\"])\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; -export const PyRerankingLinearCombinationUsage = "import lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import LinearCombinationReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = LinearCombinationReranker()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; +export const PyRerankingLinearCombinationUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import LinearCombinationReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"linear-combination-rerank\"))\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema)\ntbl.add(data)\nreranker = LinearCombinationReranker()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; -export const PyRerankingMrrUsage = "import lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import MRRReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = MRRReranker(weight_vector=0.7, weight_fts=0.3)\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n\n# Run multivector search across multiple vector columns\nrs1 = tbl.search(\"hello\").limit(10).with_row_id(True).to_arrow()\nrs2 = tbl.search(\"greeting\").limit(10).with_row_id(True).to_arrow()\ncombined = MRRReranker().rerank_multivector([rs1, rs2])\n"; +export const PyRerankingMrrUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import MRRReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"mrr-rerank\"))\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema)\ntbl.add(data)\nreranker = MRRReranker(weight_vector=0.7, weight_fts=0.3)\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n\n# Run multivector search across multiple vector columns\nrs1 = tbl.search(\"hello\").limit(10).with_row_id(True).to_arrow()\nrs2 = tbl.search(\"greeting\").limit(10).with_row_id(True).to_arrow()\ncombined = MRRReranker().rerank_multivector([rs1, rs2])\n"; export const PyRerankingOpenaiUsage = "import lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import OpenaiReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = OpenaiReranker()\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; -export const PyRerankingRrfUsage = "import lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import RRFReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = RRFReranker()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; +export const PyRerankingRrfUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import RRFReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"rrf-rerank\"))\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema)\ntbl.add(data)\nreranker = RRFReranker()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; export const PyRerankingVoyageaiUsage = "import os\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import VoyageAIReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(\"~/.lancedb\")\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema, mode=\"overwrite\")\ntbl.add(data)\nreranker = VoyageAIReranker(model_name=\"rerank-2\")\n\n# Run vector search with a reranker\nresult = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\n\n# Run FTS search with a reranker\nresult = tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n\n# Run hybrid search with a reranker\ntbl.create_fts_index(\"text\", replace=True)\nresult = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\n"; diff --git a/docs/web/snippets/materialized_view.mdx b/docs/web/snippets/materialized_view.mdx index 41f9def31..975f7635b 100644 --- a/docs/web/snippets/materialized_view.mdx +++ b/docs/web/snippets/materialized_view.mdx @@ -1,6 +1,6 @@ {/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} -export const PyMaterializedViewCreate = "import lancedb\n\ndb = lancedb.connect(\"./.lancedb\")\ntable = db.create_table(\n \"people\", [{\"name\": \"ada\", \"age\": 36}, {\"name\": \"kid\", \"age\": 7}]\n)\n\nview = db.create_materialized_view(\n \"adults\",\n \"people\",\n select=[\"name\", (\"shout\", \"upper(name)\")],\n where=\"age >= 18\",\n)\nprint(view.table.count_rows())\n# 1\n"; +export const PyMaterializedViewCreate = "import lancedb\n\ndb = lancedb.connect(\"./.lancedb\")\ntable = db.create_table(\n \"people\",\n [{\"name\": \"ada\", \"age\": 36}, {\"name\": \"kid\", \"age\": 7}],\n storage_options={\"new_table_enable_stable_row_ids\": \"true\"},\n)\n\nview = db.create_materialized_view(\n \"adults\",\n \"people\",\n select=[\"name\", (\"shout\", \"upper(name)\")],\n where=\"age >= 18\",\n)\nprint(view.table.count_rows())\n# 1\n"; -export const PyMaterializedViewRefresh = "table.add([{\"name\": \"bea\", \"age\": 41}])\n\nresult = view.refresh()\nprint(result.rows_written)\n# 2\n"; +export const PyMaterializedViewRefresh = "table.add([{\"name\": \"bea\", \"age\": 41}])\n\nview.refresh()\nprint(view.table.count_rows())\n# 2\n"; diff --git a/docs/web/snippets/provider_examples.mdx b/docs/web/snippets/provider_examples.mdx new file mode 100644 index 000000000..13768e265 --- /dev/null +++ b/docs/web/snippets/provider_examples.mdx @@ -0,0 +1,18 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const PyEmbeddingIbmUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import EmbeddingFunctionRegistry\nfrom lancedb.pydantic import LanceModel, Vector\n\nregistry = EmbeddingFunctionRegistry.get_instance()\nregistry.set_var(\"watsonx-api-key\", \"your-api-key\")\nwatsonx_embed = registry.get(\"watsonx\").create(\n name=\"ibm/slate-125m-english-rtrvr\",\n api_key=\"$var:watsonx-api-key\",\n project_id=\"your-project-id\",\n)\n\nclass TextModel(LanceModel):\n text: str = watsonx_embed.SourceField()\n vector: Vector(watsonx_embed.ndims()) = watsonx_embed.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\n\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"watsonx-demo\"))\ntbl = db.create_table(\"watsonx_test\", schema=TextModel, mode=\"overwrite\")\ntbl.add(data)\n\nrs = tbl.search(\"hello\").limit(1).to_pandas()\nprint(rs.head())\n"; + +export const PyEmbeddingJinaMultimodal = "import os\nimport tempfile\nfrom pathlib import Path\n\nimport lancedb\nimport pandas as pd\nimport requests\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\n\nos.environ[\"JINA_API_KEY\"] = \"your-api-key\"\n\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"jina-images\"))\nfunc = get_registry().get(\"jina\").create()\n\nclass Images(LanceModel):\n label: str\n image_uri: str = func.SourceField()\n image_bytes: bytes = func.SourceField()\n vector: Vector(func.ndims()) = func.VectorField()\n vec_from_bytes: Vector(func.ndims()) = func.VectorField()\n\ntable = db.create_table(\"images\", schema=Images)\nlabels = [\"cat\", \"cat\", \"dog\", \"dog\", \"horse\", \"horse\"]\nuris = [\n \"http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg\",\n \"http://farm1.staticflickr.com/134/332220238_da527d8140_z.jpg\",\n \"http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg\",\n \"http://farm5.staticflickr.com/4092/5017326486_1f46057f5f_z.jpg\",\n \"http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg\",\n \"http://farm6.staticflickr.com/5142/5835678453_4f3a4edb45_z.jpg\",\n]\nimage_bytes = [requests.get(uri).content for uri in uris]\ntable.add(\n pd.DataFrame({\"label\": labels, \"image_uri\": uris, \"image_bytes\": image_bytes})\n)\n"; + +export const PyEmbeddingJinaText = "import os\nimport tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import EmbeddingFunctionRegistry\nfrom lancedb.pydantic import LanceModel, Vector\n\nos.environ[\"JINA_API_KEY\"] = \"your-api-key\"\n\njina_embed = (\n EmbeddingFunctionRegistry.get_instance()\n .get(\"jina\")\n .create(name=\"jina-embeddings-v2-base-en\")\n)\n\nclass TextModel(LanceModel):\n text: str = jina_embed.SourceField()\n vector: Vector(jina_embed.ndims()) = jina_embed.VectorField()\n\ndata = [{\"text\": \"hello world\"}, {\"text\": \"goodbye world\"}]\n\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"jina-text\"))\ntbl = db.create_table(\"test\", schema=TextModel, mode=\"overwrite\")\n\ntbl.add(data)\n"; + +export const PyRerankingCohereUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import CohereReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"cohere-rerank\"))\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema)\ntbl.add(data)\nreranker = CohereReranker(api_key=\"your-api-key\")\n\n# Run vector search with a reranker\nvector_results = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\nprint(vector_results)\n\n# Build the full-text index before either FTS or hybrid search.\ntbl.create_fts_index(\"text\", replace=True)\n\n# Run FTS search with a reranker\nfts_results = (\n tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n)\nprint(fts_results)\n\n# Run hybrid search with a reranker\nhybrid_results = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\nprint(hybrid_results)\n"; + +export const PyRerankingJinaUsage = "import os\nimport tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import JinaReranker\n\nos.environ[\"JINA_API_KEY\"] = \"your-api-key\"\nembedder = get_registry().get(\"jina\").create()\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"jina-rerank\"))\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema)\ntbl.add(data)\nreranker = JinaReranker(api_key=\"your-api-key\")\n\n# Run vector search with a reranker\nvector_results = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\nprint(vector_results)\n\n# Build the full-text index before either FTS or hybrid search.\ntbl.create_fts_index(\"text\", replace=True)\n\n# Run FTS search with a reranker\nfts_results = (\n tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n)\nprint(fts_results)\n\n# Run hybrid search with a reranker\nhybrid_results = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\nprint(hybrid_results)\n"; + +export const PyRerankingOpenaiUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import OpenaiReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"openai-rerank\"))\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema)\ntbl.add(data)\nreranker = OpenaiReranker(api_key=\"your-api-key\")\n\n# Run vector search with a reranker\nvector_results = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\nprint(vector_results)\n\n# Build the full-text index before either FTS or hybrid search.\ntbl.create_fts_index(\"text\", replace=True)\n\n# Run FTS search with a reranker\nfts_results = (\n tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n)\nprint(fts_results)\n\n# Run hybrid search with a reranker\nhybrid_results = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\nprint(hybrid_results)\n"; + +export const PyRerankingVoyageaiUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import VoyageAIReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"voyageai-rerank\"))\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema)\ntbl.add(data)\nreranker = VoyageAIReranker(api_key=\"your-api-key\", model_name=\"rerank-2\")\n\n# Run vector search with a reranker\nvector_results = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\nprint(vector_results)\n\n# Build the full-text index before either FTS or hybrid search.\ntbl.create_fts_index(\"text\", replace=True)\n\n# Run FTS search with a reranker\nfts_results = (\n tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n)\nprint(fts_results)\n\n# Run hybrid search with a reranker\nhybrid_results = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\nprint(hybrid_results)\n"; + +export const PyRerankingWatsonxUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\nfrom lancedb.rerankers import WatsonxReranker\n\nembedder = get_registry().get(\"sentence-transformers\").create()\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"watsonx-rerank\"))\n\nclass Schema(LanceModel):\n text: str = embedder.SourceField()\n vector: Vector(embedder.ndims()) = embedder.VectorField()\n\ndata = [\n {\"text\": \"hello world\"},\n {\"text\": \"goodbye world\"},\n]\ntbl = db.create_table(\"test\", schema=Schema)\ntbl.add(data)\n\n# Supply your API key and project ID (or use space_id instead of project_id).\nreranker = WatsonxReranker(\n api_key=\"your-api-key\",\n project_id=\"your-project-id\",\n)\n\n# Run vector search with a reranker\nvector_results = tbl.search(\"hello\").rerank(reranker=reranker).to_list()\nprint(vector_results)\n\n# Build the full-text index before either FTS or hybrid search.\ntbl.create_fts_index(\"text\", replace=True)\n\n# Run FTS search with a reranker\nfts_results = (\n tbl.search(\"hello\", query_type=\"fts\").rerank(reranker=reranker).to_list()\n)\nprint(fts_results)\n\n# Run hybrid search with a reranker\nhybrid_results = (\n tbl.search(\"hello\", query_type=\"hybrid\").rerank(reranker=reranker).to_list()\n)\nprint(hybrid_results)\n"; + diff --git a/docs/web/snippets/quickstart.mdx b/docs/web/snippets/quickstart.mdx index 7a4622979..c7cf98e32 100644 --- a/docs/web/snippets/quickstart.mdx +++ b/docs/web/snippets/quickstart.mdx @@ -1,82 +1,66 @@ {/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} -export const PyQuickstartAddData = "more_data = [\n {\n \"id\": \"4\",\n \"name\": \"Morgana\",\n \"role\": \"Sorceress\",\n \"description\": \"Powerful sorceress of Avalon.\",\n \"stats\": {\"strength\": 2, \"magic\": 5, \"leadership\": 4, \"wisdom\": 4},\n \"vector\": [0.3, 0.9, 0.6, 0.8],\n \"power_score\": 3.75,\n },\n]\n\n# Add data to table\ntable.add(more_data)\n"; +export const PyQuickstartAddData = "table.add(\n [\n {\n \"id\": \"4\",\n \"name\": \"Morgana\",\n \"role\": \"Sorceress\",\n \"description\": \"Powerful sorceress of Avalon.\",\n \"stats\": {\"strength\": 2, \"magic\": 5, \"leadership\": 4, \"wisdom\": 4},\n \"vector\": [0.3, 0.9, 0.6, 0.8],\n }\n ]\n)\n"; export const PyQuickstartAddFeature = "table.add_columns(\n {\n \"power_score\": \"cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)\"\n }\n)\n"; -export const PyQuickstartComputedFeature = "table.add_columns(computed={\"combat_score\": \"(stats.magic + stats.strength) / 2.0\"})\ntable.refresh_column(\"combat_score\")\n"; +export const PyQuickstartComputedFeature = "table.add_columns(computed={\"description_length\": \"length(description)\"})\ntable.refresh_column(\"description_length\")\n"; + +export const PyQuickstartComputedFeatureEnterprise = "table.add_columns(computed={\"description_length\": \"length(description)\"})\njob = table.refresh_column_async(\"description_length\")\njob.wait()\n"; export const PyQuickstartCreateTable = "table = db.create_table(\"characters\", data=data, mode=\"overwrite\")\n"; export const PyQuickstartCreateTableAsync = "async_table = await async_db.create_table(\n \"characters\",\n data=data,\n mode=\"overwrite\",\n)\n"; -export const PyQuickstartCreateTableNoOverwrite = "table = db.create_table(\"characters\", data=data)\n"; - export const PyQuickstartCurateWithMetadata = "curated = (\n table.search(query_vector)\n .where(\"stats.magic >= 4\")\n .select([\"name\", \"role\", \"description\", \"_distance\"])\n .limit(2)\n .to_polars()\n)\nprint(curated)\n"; export const PyQuickstartData = "data = [\n {\n \"id\": \"1\",\n \"name\": \"King Arthur\",\n \"role\": \"King\",\n \"description\": \"Leader of Camelot and wielder of Excalibur.\",\n \"stats\": {\"strength\": 4, \"magic\": 1, \"leadership\": 5, \"wisdom\": 4},\n \"vector\": [0.7, 0.1, 0.9, 0.7],\n },\n {\n \"id\": \"2\",\n \"name\": \"Merlin\",\n \"role\": \"Wizard\",\n \"description\": \"Advisor and prophet with deep magical knowledge.\",\n \"stats\": {\"strength\": 2, \"magic\": 5, \"leadership\": 4, \"wisdom\": 5},\n \"vector\": [0.2, 0.9, 0.4, 0.9],\n },\n {\n \"id\": \"3\",\n \"name\": \"Sir Lancelot\",\n \"role\": \"Knight\",\n \"description\": \"Legendary knight known for courage and combat skill.\",\n \"stats\": {\"strength\": 5, \"magic\": 1, \"leadership\": 3, \"wisdom\": 3},\n \"vector\": [0.9, 0.1, 0.5, 0.4],\n },\n]\n"; -export const PyQuickstartDataAsync = "data = [\n {\n \"id\": \"1\",\n \"name\": \"King Arthur\",\n \"role\": \"King\",\n \"description\": \"Leader of Camelot and wielder of Excalibur.\",\n \"stats\": {\"strength\": 4, \"magic\": 1, \"leadership\": 5, \"wisdom\": 4},\n \"vector\": [0.7, 0.1, 0.9, 0.7],\n },\n {\n \"id\": \"2\",\n \"name\": \"Merlin\",\n \"role\": \"Wizard\",\n \"description\": \"Advisor and prophet with deep magical knowledge.\",\n \"stats\": {\"strength\": 2, \"magic\": 5, \"leadership\": 4, \"wisdom\": 5},\n \"vector\": [0.2, 0.9, 0.4, 0.9],\n },\n {\n \"id\": \"3\",\n \"name\": \"Sir Lancelot\",\n \"role\": \"Knight\",\n \"description\": \"Legendary knight known for courage and combat skill.\",\n \"stats\": {\"strength\": 5, \"magic\": 1, \"leadership\": 3, \"wisdom\": 3},\n \"vector\": [0.9, 0.1, 0.5, 0.4],\n },\n]\n"; - -export const PyQuickstartMultimodalBytes = "from pathlib import Path\n\nimage_path = Path(\"docs/web/static/assets/images/quickstart/sir-lancelot.jpg\")\nimage_bytes = image_path.read_bytes()\n\nmultimodal_table = db.create_table(\n \"character_images\",\n data=[\n {\n \"id\": \"lancelot\",\n \"description\": \"Portrait of Sir Lancelot\",\n \"image\": image_bytes,\n \"vector\": [0.9, 0.1, 0.5, 0.4],\n }\n ],\n mode=\"overwrite\",\n)\n"; - -export const PyQuickstartOpenTable = "table = db.open_table(\"characters\")\n"; +export const PyQuickstartMultimodalBytes = "from pathlib import Path\n\nimage_bytes = Path(\"data/sir-lancelot.jpg\").read_bytes()\n\nmultimodal_table = db.create_table(\n \"character_images\",\n data=[\n {\n \"id\": \"lancelot\",\n \"description\": \"Portrait of Sir Lancelot\",\n \"image\": image_bytes,\n \"vector\": [0.9, 0.1, 0.5, 0.4],\n }\n ],\n mode=\"overwrite\",\n)\n"; export const PyQuickstartOutputPandas = "# Ensure you run `pip install pandas` beforehand\nresult = table.search(query_vector).limit(2).to_pandas()\nprint(result)\n"; export const PyQuickstartQueryFeature = "features = table.search().select([\"name\", \"role\", \"power_score\"]).to_polars()\nprint(features)\n"; -export const PyQuickstartTraining = "import torch\nfrom lancedb.permutation import Permutation\n\ndataset = Permutation.identity(table).select_columns(\n [\"power_score\", \"combat_score\"]\n)\ndataloader = torch.utils.data.DataLoader(dataset, batch_size=1024, shuffle=True)\n"; - -export const PyQuickstartTrainingProjection = "dataset = Permutation.identity(table).select_columns([\"power_score\"])\ndataloader = torch.utils.data.DataLoader(dataset, batch_size=1024)\n"; +export const PyQuickstartTraining = "import torch\nfrom lancedb.permutation import Permutation\n\ntable.add_columns(\n {\n \"magic\": \"cast(stats.magic as float)\",\n \"strength\": \"cast(stats.strength as float)\",\n }\n)\ndataset = Permutation.identity(table).select_columns([\"magic\", \"strength\"])\ndataloader = torch.utils.data.DataLoader(dataset, batch_size=1024, shuffle=True)\n"; export const PyQuickstartVectorSearch1 = "# Search for examples similar to a \"wise magical advisor\"\nquery_vector = [0.2, 0.8, 0.4, 0.9]\n\n# Ensure you run `pip install polars` beforehand\nresult = (\n table.search(query_vector)\n .select([\"name\", \"role\", \"description\", \"_distance\"])\n .limit(2)\n .to_polars()\n)\nprint(result)\n"; export const PyQuickstartVectorSearch1Async = "# Search for examples similar to a \"wise magical advisor\"\nquery_vector = [0.2, 0.8, 0.4, 0.9]\n\n# Ensure you run `pip install polars` beforehand\nasync_result = await (\n await async_table.search(query_vector)\n).select([\"name\", \"role\", \"description\", \"_distance\"]).limit(2).to_polars()\nprint(async_result)\n"; -export const PyQuickstartVectorSearch2 = "# Search for examples similar to a \"powerful sorceress\"\nquery_vector = [0.3, 0.9, 0.6, 0.8]\n\nresults = table.search(query_vector).limit(2).to_polars()\nprint(results)\n"; - -export const TsQuickstartAddData = "const moreData = [\n {\n id: \"4\",\n name: \"Morgana\",\n role: \"Sorceress\",\n description: \"Powerful sorceress of Avalon.\",\n stats: { strength: 2, magic: 5, leadership: 4, wisdom: 4 },\n vector: [0.3, 0.9, 0.6, 0.8],\n power_score: 3.75,\n },\n];\n\n// Add data to table\nawait table.add(moreData);\n"; +export const TsQuickstartAddData = "await table.add([\n {\n id: \"4\",\n name: \"Morgana\",\n role: \"Sorceress\",\n description: \"Powerful sorceress of Avalon.\",\n stats: { strength: 2, magic: 5, leadership: 4, wisdom: 4 },\n vector: [0.3, 0.9, 0.6, 0.8],\n },\n]);\n"; export const TsQuickstartAddFeature = "await table.addColumns([\n {\n name: \"power_score\",\n valueSql:\n \"cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)\",\n },\n]);\n"; -export const TsQuickstartCreateTable = "let table = await db.createTable(\"characters\", data, { mode: \"overwrite\" });\n"; - -export const TsQuickstartCreateTableNoOverwrite = "table = await db.createTable(\"characters\", data);\n"; +export const TsQuickstartCreateTable = "const table = await db.createTable(\"characters\", data, { mode: \"overwrite\" });\n"; export const TsQuickstartCurateWithMetadata = "const curated = await table\n .search(queryVector)\n .where(\"stats.magic >= 4\")\n .select([\"name\", \"role\", \"description\", \"_distance\"])\n .limit(2)\n .toArray();\nconsole.table(curated);\n"; export const TsQuickstartData = "const data = [\n {\n id: \"1\",\n name: \"King Arthur\",\n role: \"King\",\n description: \"Leader of Camelot and wielder of Excalibur.\",\n stats: { strength: 4, magic: 1, leadership: 5, wisdom: 4 },\n vector: [0.7, 0.1, 0.9, 0.7],\n },\n {\n id: \"2\",\n name: \"Merlin\",\n role: \"Wizard\",\n description: \"Advisor and prophet with deep magical knowledge.\",\n stats: { strength: 2, magic: 5, leadership: 4, wisdom: 5 },\n vector: [0.2, 0.9, 0.4, 0.9],\n },\n {\n id: \"3\",\n name: \"Sir Lancelot\",\n role: \"Knight\",\n description: \"Legendary knight known for courage and combat skill.\",\n stats: { strength: 5, magic: 1, leadership: 3, wisdom: 3 },\n vector: [0.9, 0.1, 0.5, 0.4],\n },\n];\n"; -export const TsQuickstartMultimodalBytes = "const arrow = await import(\"apache-arrow\");\nconst path = await import(\"node:path\");\nconst { readFile } = await import(\"node:fs/promises\");\n\nconst imagePath = path.join(\n __dirname,\n \"../../web/static/assets/images/quickstart/sir-lancelot.jpg\",\n);\nconst imageBytes = await readFile(imagePath);\nconst imageSchema = new arrow.Schema([\n new arrow.Field(\"id\", new arrow.Utf8()),\n new arrow.Field(\"description\", new arrow.Utf8()),\n new arrow.Field(\"image\", new arrow.Binary()),\n new arrow.Field(\n \"vector\",\n new arrow.FixedSizeList(\n 4,\n new arrow.Field(\"item\", new arrow.Float32(), true),\n ),\n ),\n]);\nconst imageData = lancedb.makeArrowTable(\n [\n {\n id: \"lancelot\",\n description: \"Portrait of Sir Lancelot\",\n image: imageBytes,\n vector: [0.9, 0.1, 0.5, 0.4],\n },\n ],\n { schema: imageSchema },\n);\nconst multimodalTable = await db.createTable(\n \"character_images\",\n imageData,\n { mode: \"overwrite\" },\n);\n"; - -export const TsQuickstartOpenTable = "table = await db.openTable(\"characters\");\n"; - -export const TsQuickstartOutputArray = "result = await table.search(queryVector).limit(2).toArray();\nconsole.table(result);\n"; +export const TsQuickstartMultimodalBytes = "const arrow = await import(\"apache-arrow\");\nconst { readFile } = await import(\"node:fs/promises\");\n\nconst imageBytes = await readFile(\"data/sir-lancelot.jpg\");\nconst imageSchema = new arrow.Schema([\n new arrow.Field(\"id\", new arrow.Utf8()),\n new arrow.Field(\"description\", new arrow.Utf8()),\n new arrow.Field(\"image\", new arrow.Binary()),\n new arrow.Field(\n \"vector\",\n new arrow.FixedSizeList(\n 4,\n new arrow.Field(\"item\", new arrow.Float32(), true),\n ),\n ),\n]);\nconst imageData = lancedb.makeArrowTable(\n [\n {\n id: \"lancelot\",\n description: \"Portrait of Sir Lancelot\",\n image: imageBytes,\n vector: [0.9, 0.1, 0.5, 0.4],\n },\n ],\n { schema: imageSchema },\n);\nconst multimodalTable = await db.createTable(\n \"character_images\",\n imageData,\n { mode: \"overwrite\" },\n);\n"; export const TsQuickstartQueryFeature = "const features = await table\n .query()\n .select([\"name\", \"role\", \"power_score\"])\n .toArray();\nconsole.table(features);\n"; -export const TsQuickstartVectorSearch1 = "// Search for examples similar to a \"wise magical advisor\"\nlet queryVector = [0.2, 0.8, 0.4, 0.9];\n\nlet result = await table\n .search(queryVector)\n .select([\"name\", \"role\", \"description\", \"_distance\"])\n .limit(2)\n .toArray();\nconsole.table(result);\n"; +export const TsQuickstartVectorSearch1 = "// Search for examples similar to a \"wise magical advisor\"\nconst queryVector = [0.2, 0.8, 0.4, 0.9];\n\nconst result = await table\n .search(queryVector)\n .select([\"name\", \"role\", \"description\", \"_distance\"])\n .limit(2)\n .toArray();\nconsole.table(result);\n"; -export const TsQuickstartVectorSearch2 = "// Search for examples similar to a \"powerful sorceress\"\nqueryVector = [0.3, 0.9, 0.6, 0.8];\n\nconst results = await table.search(queryVector).limit(2).toArray();\nconsole.table(results);\n"; +export const RsQuickstartAddData = "let more_data = vec![Character {\n id: \"4\".to_string(),\n name: \"Morgana\".to_string(),\n role: \"Sorceress\".to_string(),\n description: \"Powerful sorceress of Avalon.\".to_string(),\n stats: Stats {\n strength: 2,\n magic: 5,\n leadership: 4,\n wisdom: 4,\n },\n vector: [0.3, 0.9, 0.6, 0.8],\n}];\ntable\n .add(characters_to_reader(schema.clone(), &more_data))\n .execute()\n .await\n .unwrap();\n"; export const RsQuickstartAddFeature = "table\n .add_columns(\n NewColumnTransform::SqlExpressions(vec![(\n \"power_score\".to_string(),\n \"cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)\"\n .to_string(),\n )]),\n None,\n )\n .await\n .unwrap();\n"; -export const RsQuickstartCreateTable = "let schema = characters_schema();\nlet table = db\n .create_table(\"characters\", characters_to_reader(schema.clone(), &data))\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n"; +export const RsQuickstartCreateTable = "use lancedb::database::CreateTableMode;\n\nlet schema = characters_schema();\nlet table = db\n .create_table(\"characters\", characters_to_reader(schema.clone(), &data))\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n"; -export const RsQuickstartCreateTableNoOverwrite = "let table = db\n .create_table(\"characters\", characters_to_reader(schema.clone(), &data))\n .execute()\n .await\n .unwrap();\n"; - -export const RsQuickstartCurateWithMetadata = "let curated: DataFrame = table\n .query()\n .nearest_to(&query_vector)\n .unwrap()\n .only_if(\"stats.magic >= 4\")\n .select(Select::Columns(vec![\n \"name\".to_string(),\n \"role\".to_string(),\n \"description\".to_string(),\n \"_distance\".to_string(),\n ]))\n .limit(2)\n .execute()\n .await\n .unwrap()\n .into_polars()\n .await\n .unwrap();\nprintln!(\"{curated:?}\");\n"; +export const RsQuickstartCurateWithMetadata = "let curated = table\n .query()\n .nearest_to(&query_vector)\n .unwrap()\n .only_if(\"stats.magic >= 4\")\n .select(Select::Columns(vec![\n \"name\".to_string(),\n \"role\".to_string(),\n \"description\".to_string(),\n \"_distance\".to_string(),\n ]))\n .limit(2)\n .execute()\n .await\n .unwrap()\n .into_polars()\n .await\n .unwrap();\nprintln!(\"{curated:?}\");\n"; export const RsQuickstartData = "let data = vec![\n Character {\n id: \"1\".to_string(),\n name: \"King Arthur\".to_string(),\n role: \"King\".to_string(),\n description: \"Leader of Camelot and wielder of Excalibur.\".to_string(),\n stats: Stats {\n strength: 4,\n magic: 1,\n leadership: 5,\n wisdom: 4,\n },\n vector: [0.7, 0.1, 0.9, 0.7],\n },\n Character {\n id: \"2\".to_string(),\n name: \"Merlin\".to_string(),\n role: \"Wizard\".to_string(),\n description: \"Advisor and prophet with deep magical knowledge.\".to_string(),\n stats: Stats {\n strength: 2,\n magic: 5,\n leadership: 4,\n wisdom: 5,\n },\n vector: [0.2, 0.9, 0.4, 0.9],\n },\n Character {\n id: \"3\".to_string(),\n name: \"Sir Lancelot\".to_string(),\n role: \"Knight\".to_string(),\n description: \"Legendary knight known for courage and combat skill.\".to_string(),\n stats: Stats {\n strength: 5,\n magic: 1,\n leadership: 3,\n wisdom: 3,\n },\n vector: [0.9, 0.1, 0.5, 0.4],\n },\n];\n"; -export const RsQuickstartDefineStruct = "// Define structs representing the data schema\n#[derive(Debug, Clone, Serialize, Deserialize)]\nstruct Stats {\n strength: i8,\n magic: i8,\n leadership: i8,\n wisdom: i8,\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\nstruct Character {\n id: String,\n name: String,\n role: String,\n description: String,\n stats: Stats,\n vector: [f32; 4],\n}\n\nfn characters_schema() -> Arc {\n Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::LargeUtf8, false),\n Field::new(\"name\", DataType::LargeUtf8, false),\n Field::new(\"role\", DataType::LargeUtf8, false),\n Field::new(\"description\", DataType::LargeUtf8, false),\n Field::new(\n \"stats\",\n DataType::Struct(arrow_schema::Fields::from(vec![\n Arc::new(Field::new(\"strength\", DataType::Int8, false)),\n Arc::new(Field::new(\"magic\", DataType::Int8, false)),\n Arc::new(Field::new(\"leadership\", DataType::Int8, false)),\n Arc::new(Field::new(\"wisdom\", DataType::Int8, false)),\n ])),\n false,\n ),\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 4),\n false,\n ),\n ]))\n}\n"; +export const RsQuickstartDefineStruct = "use std::sync::Arc;\n\nuse lancedb::arrow::arrow_array::types::Float32Type;\nuse lancedb::arrow::arrow_array::{\n FixedSizeListArray, Int8Array, LargeStringArray, RecordBatch, RecordBatchIterator,\n RecordBatchReader, StructArray,\n};\nuse lancedb::arrow::arrow_schema::{DataType, Field, FieldRef, Fields, Schema};\n\n// Define structs representing the data schema\n#[derive(Debug, Clone)]\nstruct Stats {\n strength: i8,\n magic: i8,\n leadership: i8,\n wisdom: i8,\n}\n\n#[derive(Debug, Clone)]\nstruct Character {\n id: String,\n name: String,\n role: String,\n description: String,\n stats: Stats,\n vector: [f32; 4],\n}\n\nfn characters_schema() -> Arc {\n Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::LargeUtf8, false),\n Field::new(\"name\", DataType::LargeUtf8, false),\n Field::new(\"role\", DataType::LargeUtf8, false),\n Field::new(\"description\", DataType::LargeUtf8, false),\n Field::new(\n \"stats\",\n DataType::Struct(Fields::from(vec![\n Arc::new(Field::new(\"strength\", DataType::Int8, false)),\n Arc::new(Field::new(\"magic\", DataType::Int8, false)),\n Arc::new(Field::new(\"leadership\", DataType::Int8, false)),\n Arc::new(Field::new(\"wisdom\", DataType::Int8, false)),\n ])),\n false,\n ),\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 4),\n false,\n ),\n ]))\n}\n\nfn characters_to_reader(\n schema: Arc,\n rows: &[Character],\n) -> Box {\n let ids = LargeStringArray::from_iter_values(rows.iter().map(|row| row.id.as_str()));\n let names = LargeStringArray::from_iter_values(rows.iter().map(|row| row.name.as_str()));\n let roles = LargeStringArray::from_iter_values(rows.iter().map(|row| row.role.as_str()));\n let descriptions =\n LargeStringArray::from_iter_values(rows.iter().map(|row| row.description.as_str()));\n\n let strength = Int8Array::from_iter_values(rows.iter().map(|row| row.stats.strength));\n let magic = Int8Array::from_iter_values(rows.iter().map(|row| row.stats.magic));\n let leadership = Int8Array::from_iter_values(rows.iter().map(|row| row.stats.leadership));\n let wisdom = Int8Array::from_iter_values(rows.iter().map(|row| row.stats.wisdom));\n let stats_fields: Vec = vec![\n Arc::new(Field::new(\"strength\", DataType::Int8, false)),\n Arc::new(Field::new(\"magic\", DataType::Int8, false)),\n Arc::new(Field::new(\"leadership\", DataType::Int8, false)),\n Arc::new(Field::new(\"wisdom\", DataType::Int8, false)),\n ];\n let stats = StructArray::new(\n stats_fields.into(),\n vec![\n Arc::new(strength),\n Arc::new(magic),\n Arc::new(leadership),\n Arc::new(wisdom),\n ],\n None,\n );\n\n let vectors = FixedSizeListArray::from_iter_primitive::(\n rows.iter()\n .map(|row| Some(row.vector.iter().copied().map(Some).collect::>())),\n 4,\n );\n\n let batch = RecordBatch::try_new(\n schema.clone(),\n vec![\n Arc::new(ids),\n Arc::new(names),\n Arc::new(roles),\n Arc::new(descriptions),\n Arc::new(stats),\n Arc::new(vectors),\n ],\n )\n .unwrap();\n\n Box::new(RecordBatchIterator::new(\n vec![Ok(batch)].into_iter(),\n schema,\n ))\n}\n"; -export const RsQuickstartMultimodalBytes = "use std::sync::Arc;\n\nuse arrow_array::{\n BinaryArray, FixedSizeListArray, LargeStringArray, RecordBatch, RecordBatchIterator,\n};\nuse arrow_schema::{DataType, Field, Schema};\n\nlet image_path = std::path::Path::new(env!(\"CARGO_MANIFEST_DIR\"))\n .join(\"../../web/static/assets/images/quickstart/sir-lancelot.jpg\");\nlet image_bytes = std::fs::read(image_path).unwrap();\n\nlet image_schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::LargeUtf8, false),\n Field::new(\"description\", DataType::LargeUtf8, false),\n Field::new(\"image\", DataType::Binary, false),\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 4),\n false,\n ),\n]));\nlet image_vectors = [[0.9_f32, 0.1, 0.5, 0.4]];\nlet image_batch = RecordBatch::try_new(\n image_schema.clone(),\n vec![\n Arc::new(LargeStringArray::from_iter_values([\"lancelot\"])),\n Arc::new(LargeStringArray::from_iter_values([\n \"Portrait of Sir Lancelot\",\n ])),\n Arc::new(BinaryArray::from_iter_values([image_bytes.as_slice()])),\n Arc::new(\n FixedSizeListArray::from_iter_primitive::(\n image_vectors\n .iter()\n .map(|vector| Some(vector.iter().copied().map(Some).collect::>())),\n 4,\n ),\n ),\n ],\n)\n.unwrap();\nlet image_reader: Box = Box::new(\n RecordBatchIterator::new(vec![Ok(image_batch)].into_iter(), image_schema),\n);\nlet multimodal_table = db\n .create_table(\"character_images\", image_reader)\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n"; +export const RsQuickstartMultimodalBytes = "use lancedb::arrow::arrow_array::BinaryArray;\n\nlet image_bytes = std::fs::read(\"data/sir-lancelot.jpg\").unwrap();\n\nlet image_schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::LargeUtf8, false),\n Field::new(\"description\", DataType::LargeUtf8, false),\n Field::new(\"image\", DataType::Binary, false),\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 4),\n false,\n ),\n]));\nlet image_vectors = [[0.9_f32, 0.1, 0.5, 0.4]];\nlet image_batch = RecordBatch::try_new(\n image_schema.clone(),\n vec![\n Arc::new(LargeStringArray::from_iter_values([\"lancelot\"])),\n Arc::new(LargeStringArray::from_iter_values([\n \"Portrait of Sir Lancelot\",\n ])),\n Arc::new(BinaryArray::from_iter_values([image_bytes.as_slice()])),\n Arc::new(\n FixedSizeListArray::from_iter_primitive::(\n image_vectors\n .iter()\n .map(|vector| Some(vector.iter().copied().map(Some).collect::>())),\n 4,\n ),\n ),\n ],\n)\n.unwrap();\nlet image_reader: Box = Box::new(RecordBatchIterator::new(\n vec![Ok(image_batch)].into_iter(),\n image_schema,\n));\nlet multimodal_table = db\n .create_table(\"character_images\", image_reader)\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n"; -export const RsQuickstartOutputArray = "let result: DataFrame = table\n .query()\n .nearest_to(&query_vector)\n .unwrap()\n .select(Select::Columns(vec![\n \"name\".to_string(),\n \"role\".to_string(),\n \"description\".to_string(),\n \"_distance\".to_string(),\n ]))\n .limit(2)\n .execute()\n .await\n .unwrap()\n .into_polars()\n .await\n .unwrap();\nprintln!(\"{result:?}\");\n"; +export const RsQuickstartOutputArray = "let result = table\n .query()\n .nearest_to(&query_vector)\n .unwrap()\n .select(Select::Columns(vec![\n \"name\".to_string(),\n \"role\".to_string(),\n \"description\".to_string(),\n \"_distance\".to_string(),\n ]))\n .limit(2)\n .execute()\n .await\n .unwrap()\n .into_polars()\n .await\n .unwrap();\nprintln!(\"{result:?}\");\n"; -export const RsQuickstartQueryFeature = "let features: DataFrame = table\n .query()\n .select(Select::Columns(vec![\n \"name\".to_string(),\n \"role\".to_string(),\n \"power_score\".to_string(),\n ]))\n .execute()\n .await\n .unwrap()\n .into_polars()\n .await\n .unwrap();\nprintln!(\"{features:?}\");\n"; +export const RsQuickstartQueryFeature = "let features = table\n .query()\n .select(Select::Columns(vec![\n \"name\".to_string(),\n \"role\".to_string(),\n \"power_score\".to_string(),\n ]))\n .execute()\n .await\n .unwrap()\n .into_polars()\n .await\n .unwrap();\nprintln!(\"{features:?}\");\n"; -export const RsQuickstartVectorSearch1 = "// Search for examples similar to a \"wise magical advisor\"\nlet query_vector = [0.2, 0.8, 0.4, 0.9];\n\nlet result: DataFrame = table\n .query()\n .nearest_to(&query_vector)\n .unwrap()\n .select(Select::Columns(vec![\n \"name\".to_string(),\n \"role\".to_string(),\n \"description\".to_string(),\n \"_distance\".to_string(),\n ]))\n .limit(2)\n .execute()\n .await\n .unwrap()\n .into_polars()\n .await\n .unwrap();\nprintln!(\"{result:?}\");\n"; +export const RsQuickstartVectorSearch1 = "use lancedb::arrow::IntoPolars;\nuse lancedb::query::{ExecutableQuery, QueryBase, Select};\n\n// Search for examples similar to a \"wise magical advisor\"\nlet query_vector = [0.2, 0.8, 0.4, 0.9];\n\nlet result = table\n .query()\n .nearest_to(&query_vector)\n .unwrap()\n .select(Select::Columns(vec![\n \"name\".to_string(),\n \"role\".to_string(),\n \"description\".to_string(),\n \"_distance\".to_string(),\n ]))\n .limit(2)\n .execute()\n .await\n .unwrap()\n .into_polars()\n .await\n .unwrap();\nprintln!(\"{result:?}\");\n"; diff --git a/docs/web/snippets/read_consistency.mdx b/docs/web/snippets/read_consistency.mdx new file mode 100644 index 000000000..f99521a4d --- /dev/null +++ b/docs/web/snippets/read_consistency.mdx @@ -0,0 +1,4 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const PyReadConsistency = "from datetime import timedelta\nfrom tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-consistency-\") as uri:\n writer = lancedb.connect(uri)\n table = writer.create_table(\"documents\", [{\"id\": 1, \"text\": \"first\"}])\n\n default_reader = lancedb.connect(uri).open_table(\"documents\")\n latest_reader = lancedb.connect(\n uri, read_consistency_interval=timedelta(0)\n ).open_table(\"documents\")\n\n table.add([{\"id\": 2, \"text\": \"second\"}])\n\n default_count = default_reader.count_rows()\n latest_count = latest_reader.count_rows()\n print(f\"Default reader: {default_count} rows\") # 1\n print(f\"Zero-interval reader: {latest_count} rows\") # 2\n\n default_reader.checkout_latest()\n refreshed_count = default_reader.count_rows()\n print(f\"After explicit refresh: {refreshed_count} rows\") # 2\n"; + diff --git a/docs/web/snippets/search_examples.mdx b/docs/web/snippets/search_examples.mdx new file mode 100644 index 000000000..a9aeb5e7f --- /dev/null +++ b/docs/web/snippets/search_examples.mdx @@ -0,0 +1,32 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const PyFilterSearch = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-search-\") as uri:\n db = lancedb.connect(uri)\n table = db.create_table(\n \"documents\",\n [\n {\n \"id\": 1,\n \"text\": \"A puppy playing in the park\",\n \"category\": \"animals\",\n \"vector\": [1.0, 0.0],\n },\n {\n \"id\": 2,\n \"text\": \"A dog resting at home\",\n \"category\": \"animals\",\n \"vector\": [0.9, 0.1],\n },\n {\n \"id\": 3,\n \"text\": \"A guide to growing tomatoes\",\n \"category\": \"gardening\",\n \"vector\": [0.0, 1.0],\n },\n ],\n )\n rows = (\n table.search([0.0, 1.0])\n .where(\"category = 'animals'\", prefilter=True)\n .select([\"id\", \"text\", \"category\", \"_distance\"])\n .limit(2)\n .to_list()\n )\n print(rows)\n"; + +export const PyHybridSearch = "from tempfile import TemporaryDirectory\n\nimport lancedb\nfrom lancedb.index import FTS\n\nwith TemporaryDirectory(prefix=\"lancedb-search-\") as uri:\n db = lancedb.connect(uri)\n table = db.create_table(\n \"documents\",\n [\n {\n \"id\": 1,\n \"text\": \"A puppy playing in the park\",\n \"category\": \"animals\",\n \"vector\": [1.0, 0.0],\n },\n {\n \"id\": 2,\n \"text\": \"A dog resting at home\",\n \"category\": \"animals\",\n \"vector\": [0.9, 0.1],\n },\n {\n \"id\": 3,\n \"text\": \"A guide to growing tomatoes\",\n \"category\": \"gardening\",\n \"vector\": [0.0, 1.0],\n },\n ],\n )\n from lancedb.rerankers import RRFReranker\n\n table.create_index(\"text\", config=FTS())\n rows = (\n table.search(\n query_type=\"hybrid\", vector_column_name=\"vector\", fts_columns=\"text\"\n )\n .vector([1.0, 0.0])\n .text(\"puppy\")\n .rerank(RRFReranker())\n .limit(2)\n .to_list()\n )\n print(rows)\n"; + +export const PyQueryPlan = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-search-\") as uri:\n db = lancedb.connect(uri)\n table = db.create_table(\n \"documents\",\n [\n {\n \"id\": 1,\n \"text\": \"A puppy playing in the park\",\n \"category\": \"animals\",\n \"vector\": [1.0, 0.0],\n },\n {\n \"id\": 2,\n \"text\": \"A dog resting at home\",\n \"category\": \"animals\",\n \"vector\": [0.9, 0.1],\n },\n {\n \"id\": 3,\n \"text\": \"A guide to growing tomatoes\",\n \"category\": \"gardening\",\n \"vector\": [0.0, 1.0],\n },\n ],\n )\n query = table.search([1.0, 0.0]).select([\"id\", \"text\", \"_distance\"]).limit(2)\n print(\"Plan:\", query.explain_plan())\n print(\"Measured execution:\", query.analyze_plan())\n rows = query.to_list()\n print(rows)\n"; + +export const PyTextSearch = "from tempfile import TemporaryDirectory\n\nimport lancedb\nfrom lancedb.index import FTS\n\nwith TemporaryDirectory(prefix=\"lancedb-search-\") as uri:\n db = lancedb.connect(uri)\n table = db.create_table(\n \"documents\",\n [\n {\n \"id\": 1,\n \"text\": \"A puppy playing in the park\",\n \"category\": \"animals\",\n },\n {\n \"id\": 2,\n \"text\": \"A dog resting at home\",\n \"category\": \"animals\",\n },\n {\n \"id\": 3,\n \"text\": \"A guide to growing tomatoes\",\n \"category\": \"gardening\",\n },\n ],\n )\n table.create_index(\"text\", config=FTS())\n rows = (\n table.search(\"puppy\", query_type=\"fts\", fts_columns=\"text\")\n .select([\"id\", \"text\", \"_score\"])\n .limit(2)\n .to_list()\n )\n print(rows)\n"; + +export const PyVectorSearch = "from tempfile import TemporaryDirectory\n\nimport lancedb\n\nwith TemporaryDirectory(prefix=\"lancedb-search-\") as uri:\n db = lancedb.connect(uri)\n table = db.create_table(\n \"documents\",\n [\n {\n \"id\": 1,\n \"text\": \"A puppy playing in the park\",\n \"category\": \"animals\",\n \"vector\": [1.0, 0.0],\n },\n {\n \"id\": 2,\n \"text\": \"A dog resting at home\",\n \"category\": \"animals\",\n \"vector\": [0.9, 0.1],\n },\n {\n \"id\": 3,\n \"text\": \"A guide to growing tomatoes\",\n \"category\": \"gardening\",\n \"vector\": [0.0, 1.0],\n },\n ],\n )\n rows = (\n table.search([1.0, 0.0])\n .distance_type(\"l2\")\n .select([\"id\", \"text\", \"_distance\"])\n .limit(2)\n .to_list()\n )\n print(rows)\n"; + +export const TsFilterSearchBody = "const lancedb = await import(\"@lancedb/lancedb\");\nconst { mkdtemp, rm } = await import(\"node:fs/promises\");\nconst { tmpdir } = await import(\"node:os\");\nconst { join } = await import(\"node:path\");\nconst uri = await mkdtemp(join(tmpdir(), \"lancedb-search-\"));\ntry {\n const db = await lancedb.connect(uri);\n const table = await db.createTable(\"documents\", [\n {\n id: 1,\n text: \"A puppy playing in the park\",\n category: \"animals\",\n vector: [1.0, 0.0],\n },\n {\n id: 2,\n text: \"A dog resting at home\",\n category: \"animals\",\n vector: [0.9, 0.1],\n },\n {\n id: 3,\n text: \"A guide to growing tomatoes\",\n category: \"gardening\",\n vector: [0.0, 1.0],\n },\n ]);\n const rows = await table\n .vectorSearch([0.0, 1.0])\n .where(\"category = 'animals'\")\n .select([\"id\", \"text\", \"category\", \"_distance\"])\n .limit(2)\n .toArray();\n console.log(rows);\n"; + +export const TsFilterSearchTail = "} finally {\n await rm(uri, { recursive: true, force: true });\n}\n"; + +export const TsHybridSearchBody = "const lancedb = await import(\"@lancedb/lancedb\");\nconst { mkdtemp, rm } = await import(\"node:fs/promises\");\nconst { tmpdir } = await import(\"node:os\");\nconst { join } = await import(\"node:path\");\nconst uri = await mkdtemp(join(tmpdir(), \"lancedb-search-\"));\ntry {\n const db = await lancedb.connect(uri);\n const table = await db.createTable(\"documents\", [\n {\n id: 1,\n text: \"A puppy playing in the park\",\n category: \"animals\",\n vector: [1.0, 0.0],\n },\n {\n id: 2,\n text: \"A dog resting at home\",\n category: \"animals\",\n vector: [0.9, 0.1],\n },\n {\n id: 3,\n text: \"A guide to growing tomatoes\",\n category: \"gardening\",\n vector: [0.0, 1.0],\n },\n ]);\n await table.createIndex(\"text\", { config: lancedb.Index.fts() });\n const reranker = await lancedb.rerankers.RRFReranker.create();\n const rows = await table\n .vectorSearch([1.0, 0.0])\n .fullTextSearch(\"puppy\", { columns: [\"text\"] })\n .rerank(reranker)\n .limit(2)\n .toArray();\n console.log(rows);\n"; + +export const TsHybridSearchTail = "} finally {\n await rm(uri, { recursive: true, force: true });\n}\n"; + +export const TsQueryPlanBody = "const lancedb = await import(\"@lancedb/lancedb\");\nconst { mkdtemp, rm } = await import(\"node:fs/promises\");\nconst { tmpdir } = await import(\"node:os\");\nconst { join } = await import(\"node:path\");\nconst uri = await mkdtemp(join(tmpdir(), \"lancedb-search-\"));\ntry {\n const db = await lancedb.connect(uri);\n const table = await db.createTable(\"documents\", [\n {\n id: 1,\n text: \"A puppy playing in the park\",\n category: \"animals\",\n vector: [1.0, 0.0],\n },\n {\n id: 2,\n text: \"A dog resting at home\",\n category: \"animals\",\n vector: [0.9, 0.1],\n },\n {\n id: 3,\n text: \"A guide to growing tomatoes\",\n category: \"gardening\",\n vector: [0.0, 1.0],\n },\n ]);\n const query = table\n .vectorSearch([1.0, 0.0])\n .select([\"id\", \"text\", \"_distance\"])\n .limit(2);\n console.log(\"Plan:\", await query.explainPlan());\n console.log(\"Measured execution:\", await query.analyzePlan());\n const rows = await query.toArray();\n console.log(rows);\n"; + +export const TsQueryPlanTail = "} finally {\n await rm(uri, { recursive: true, force: true });\n}\n"; + +export const TsTextSearchBody = "const lancedb = await import(\"@lancedb/lancedb\");\nconst { mkdtemp, rm } = await import(\"node:fs/promises\");\nconst { tmpdir } = await import(\"node:os\");\nconst { join } = await import(\"node:path\");\nconst uri = await mkdtemp(join(tmpdir(), \"lancedb-search-\"));\ntry {\n const db = await lancedb.connect(uri);\n const table = await db.createTable(\"documents\", [\n { id: 1, text: \"A puppy playing in the park\", category: \"animals\" },\n { id: 2, text: \"A dog resting at home\", category: \"animals\" },\n { id: 3, text: \"A guide to growing tomatoes\", category: \"gardening\" },\n ]);\n await table.createIndex(\"text\", { config: lancedb.Index.fts() });\n const rows = await table\n .query()\n .fullTextSearch(\"puppy\", { columns: [\"text\"] })\n .select([\"id\", \"text\", \"_score\"])\n .limit(2)\n .toArray();\n console.log(rows);\n"; + +export const TsTextSearchTail = "} finally {\n await rm(uri, { recursive: true, force: true });\n}\n"; + +export const TsVectorSearchBody = "const lancedb = await import(\"@lancedb/lancedb\");\nconst { mkdtemp, rm } = await import(\"node:fs/promises\");\nconst { tmpdir } = await import(\"node:os\");\nconst { join } = await import(\"node:path\");\nconst uri = await mkdtemp(join(tmpdir(), \"lancedb-search-\"));\ntry {\n const db = await lancedb.connect(uri);\n const table = await db.createTable(\"documents\", [\n {\n id: 1,\n text: \"A puppy playing in the park\",\n category: \"animals\",\n vector: [1.0, 0.0],\n },\n {\n id: 2,\n text: \"A dog resting at home\",\n category: \"animals\",\n vector: [0.9, 0.1],\n },\n {\n id: 3,\n text: \"A guide to growing tomatoes\",\n category: \"gardening\",\n vector: [0.0, 1.0],\n },\n ]);\n const rows = await table\n .vectorSearch([1.0, 0.0])\n .distanceType(\"l2\")\n .select([\"id\", \"text\", \"_distance\"])\n .limit(2)\n .toArray();\n console.log(rows);\n"; + +export const TsVectorSearchTail = "} finally {\n await rm(uri, { recursive: true, force: true });\n}\n"; + diff --git a/docs/web/snippets/search_filter_search.mdx b/docs/web/snippets/search_filter_search.mdx new file mode 100644 index 000000000..efe325aae --- /dev/null +++ b/docs/web/snippets/search_filter_search.mdx @@ -0,0 +1,6 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const RsExampleBody = "use arrow_array::types::Float32Type;\nuse arrow_array::{FixedSizeListArray, Int64Array, RecordBatch, RecordBatchIterator, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::TryStreamExt;\nuse lancedb::query::{ExecutableQuery, QueryBase, Select};\nuse std::sync::Arc;\n\n#[tokio::main]\nasync fn main() -> Result<(), Box> {\n let directory = tempfile::tempdir()?;\n let db = lancedb::connect(directory.path().to_str().unwrap())\n .execute()\n .await?;\n let schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"text\", DataType::Utf8, false),\n Field::new(\"category\", DataType::Utf8, false),\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 2),\n true,\n ),\n ]));\n let batch = RecordBatch::try_new(\n schema.clone(),\n vec![\n Arc::new(Int64Array::from(vec![1, 2, 3])),\n Arc::new(StringArray::from(vec![\n \"A puppy playing in the park\",\n \"A dog resting at home\",\n \"A guide to growing tomatoes\",\n ])),\n Arc::new(StringArray::from(vec![\"animals\", \"animals\", \"gardening\"])),\n Arc::new(\n FixedSizeListArray::from_iter_primitive::(\n [[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]]\n .into_iter()\n .map(|v| Some(v.into_iter().map(Some).collect::>())),\n 2,\n ),\n ),\n ],\n )?;\n let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema);\n let table = db\n .create_table(\n \"documents\",\n Box::new(reader) as Box,\n )\n .execute()\n .await?;\n let query = table\n .vector_search(&[0.0_f32, 1.0])?\n .only_if(\"category = 'animals'\")\n .select(Select::columns(&[\"id\", \"text\", \"category\", \"_distance\"]))\n .limit(2);\n let rows: Vec = query.execute().await?.try_collect().await?;\n println!(\"{rows:?}\");\n"; + +export const RsExampleTail = " Ok(())\n}\n"; + diff --git a/docs/web/snippets/search_hybrid_search.mdx b/docs/web/snippets/search_hybrid_search.mdx new file mode 100644 index 000000000..492e14f54 --- /dev/null +++ b/docs/web/snippets/search_hybrid_search.mdx @@ -0,0 +1,6 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const RsExampleBody = "use arrow_array::types::Float32Type;\nuse arrow_array::{FixedSizeListArray, Int64Array, RecordBatch, RecordBatchIterator, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::TryStreamExt;\nuse lance_index::scalar::FullTextSearchQuery;\nuse lancedb::index::{Index, scalar::FtsIndexBuilder};\nuse lancedb::query::{ExecutableQuery, QueryBase};\nuse std::sync::Arc;\n\n#[tokio::main]\nasync fn main() -> Result<(), Box> {\n let directory = tempfile::tempdir()?;\n let db = lancedb::connect(directory.path().to_str().unwrap())\n .execute()\n .await?;\n let schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"text\", DataType::Utf8, false),\n Field::new(\"category\", DataType::Utf8, false),\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 2),\n true,\n ),\n ]));\n let batch = RecordBatch::try_new(\n schema.clone(),\n vec![\n Arc::new(Int64Array::from(vec![1, 2, 3])),\n Arc::new(StringArray::from(vec![\n \"A puppy playing in the park\",\n \"A dog resting at home\",\n \"A guide to growing tomatoes\",\n ])),\n Arc::new(StringArray::from(vec![\"animals\", \"animals\", \"gardening\"])),\n Arc::new(\n FixedSizeListArray::from_iter_primitive::(\n [[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]]\n .into_iter()\n .map(|v| Some(v.into_iter().map(Some).collect::>())),\n 2,\n ),\n ),\n ],\n )?;\n let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema);\n let table = db\n .create_table(\n \"documents\",\n Box::new(reader) as Box,\n )\n .execute()\n .await?;\n table\n .create_index(&[\"text\"], Index::FTS(FtsIndexBuilder::default()))\n .execute()\n .await?;\n let query = table\n .vector_search(&[1.0_f32, 0.0])?\n .full_text_search(FullTextSearchQuery::new(\"puppy\".into()).with_column(\"text\".into())?)\n .rerank(Arc::new(lancedb::rerankers::rrf::RRFReranker::default()))\n .limit(2);\n let rows: Vec = query.execute().await?.try_collect().await?;\n println!(\"{rows:?}\");\n"; + +export const RsExampleTail = " Ok(())\n}\n"; + diff --git a/docs/web/snippets/search_query_plan.mdx b/docs/web/snippets/search_query_plan.mdx new file mode 100644 index 000000000..fbfe156c9 --- /dev/null +++ b/docs/web/snippets/search_query_plan.mdx @@ -0,0 +1,6 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const RsExampleBody = "use arrow_array::types::Float32Type;\nuse arrow_array::{FixedSizeListArray, Int64Array, RecordBatch, RecordBatchIterator, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::TryStreamExt;\nuse lancedb::query::{ExecutableQuery, QueryBase, Select};\nuse std::sync::Arc;\n\n#[tokio::main]\nasync fn main() -> Result<(), Box> {\n let directory = tempfile::tempdir()?;\n let db = lancedb::connect(directory.path().to_str().unwrap())\n .execute()\n .await?;\n let schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"text\", DataType::Utf8, false),\n Field::new(\"category\", DataType::Utf8, false),\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 2),\n true,\n ),\n ]));\n let batch = RecordBatch::try_new(\n schema.clone(),\n vec![\n Arc::new(Int64Array::from(vec![1, 2, 3])),\n Arc::new(StringArray::from(vec![\n \"A puppy playing in the park\",\n \"A dog resting at home\",\n \"A guide to growing tomatoes\",\n ])),\n Arc::new(StringArray::from(vec![\"animals\", \"animals\", \"gardening\"])),\n Arc::new(\n FixedSizeListArray::from_iter_primitive::(\n [[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]]\n .into_iter()\n .map(|v| Some(v.into_iter().map(Some).collect::>())),\n 2,\n ),\n ),\n ],\n )?;\n let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema);\n let table = db\n .create_table(\n \"documents\",\n Box::new(reader) as Box,\n )\n .execute()\n .await?;\n let query = table\n .vector_search(&[1.0_f32, 0.0])?\n .select(Select::columns(&[\"id\", \"text\", \"_distance\"]))\n .limit(2);\n println!(\"Plan: {}\", query.explain_plan(false).await?);\n println!(\"Measured execution: {}\", query.analyze_plan().await?);\n let rows: Vec = query.execute().await?.try_collect().await?;\n println!(\"{rows:?}\");\n"; + +export const RsExampleTail = " Ok(())\n}\n"; + diff --git a/docs/web/snippets/search_text_search.mdx b/docs/web/snippets/search_text_search.mdx new file mode 100644 index 000000000..86745b652 --- /dev/null +++ b/docs/web/snippets/search_text_search.mdx @@ -0,0 +1,6 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const RsExampleBody = "use arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::TryStreamExt;\nuse lance_index::scalar::FullTextSearchQuery;\nuse lancedb::index::{Index, scalar::FtsIndexBuilder};\nuse lancedb::query::{ExecutableQuery, QueryBase, Select};\nuse std::sync::Arc;\n\n#[tokio::main]\nasync fn main() -> Result<(), Box> {\n let directory = tempfile::tempdir()?;\n let db = lancedb::connect(directory.path().to_str().unwrap())\n .execute()\n .await?;\n let schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"text\", DataType::Utf8, false),\n Field::new(\"category\", DataType::Utf8, false),\n ]));\n let batch = RecordBatch::try_new(\n schema.clone(),\n vec![\n Arc::new(Int64Array::from(vec![1, 2, 3])),\n Arc::new(StringArray::from(vec![\n \"A puppy playing in the park\",\n \"A dog resting at home\",\n \"A guide to growing tomatoes\",\n ])),\n Arc::new(StringArray::from(vec![\"animals\", \"animals\", \"gardening\"])),\n ],\n )?;\n let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema);\n let table = db\n .create_table(\n \"documents\",\n Box::new(reader) as Box,\n )\n .execute()\n .await?;\n table\n .create_index(&[\"text\"], Index::FTS(FtsIndexBuilder::default()))\n .execute()\n .await?;\n let query = table\n .query()\n .full_text_search(FullTextSearchQuery::new(\"puppy\".into()).with_column(\"text\".into())?)\n .select(Select::columns(&[\"id\", \"text\", \"_score\"]))\n .limit(2);\n let rows: Vec = query.execute().await?.try_collect().await?;\n println!(\"{rows:?}\");\n"; + +export const RsExampleTail = " Ok(())\n}\n"; + diff --git a/docs/web/snippets/search_vector_search.mdx b/docs/web/snippets/search_vector_search.mdx new file mode 100644 index 000000000..47966f5d4 --- /dev/null +++ b/docs/web/snippets/search_vector_search.mdx @@ -0,0 +1,6 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const RsExampleBody = "use arrow_array::types::Float32Type;\nuse arrow_array::{FixedSizeListArray, Int64Array, RecordBatch, RecordBatchIterator, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::TryStreamExt;\nuse lancedb::query::{ExecutableQuery, QueryBase, Select};\nuse std::sync::Arc;\n\n#[tokio::main]\nasync fn main() -> Result<(), Box> {\n let directory = tempfile::tempdir()?;\n let db = lancedb::connect(directory.path().to_str().unwrap())\n .execute()\n .await?;\n let schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\"text\", DataType::Utf8, false),\n Field::new(\"category\", DataType::Utf8, false),\n Field::new(\n \"vector\",\n DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 2),\n true,\n ),\n ]));\n let batch = RecordBatch::try_new(\n schema.clone(),\n vec![\n Arc::new(Int64Array::from(vec![1, 2, 3])),\n Arc::new(StringArray::from(vec![\n \"A puppy playing in the park\",\n \"A dog resting at home\",\n \"A guide to growing tomatoes\",\n ])),\n Arc::new(StringArray::from(vec![\"animals\", \"animals\", \"gardening\"])),\n Arc::new(\n FixedSizeListArray::from_iter_primitive::(\n [[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]]\n .into_iter()\n .map(|v| Some(v.into_iter().map(Some).collect::>())),\n 2,\n ),\n ),\n ],\n )?;\n let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema);\n let table = db\n .create_table(\n \"documents\",\n Box::new(reader) as Box,\n )\n .execute()\n .await?;\n let query = table\n .vector_search(&[1.0_f32, 0.0])?\n .distance_type(lancedb::DistanceType::L2)\n .select(Select::columns(&[\"id\", \"text\", \"_distance\"]))\n .limit(2);\n let rows: Vec = query.execute().await?.try_collect().await?;\n println!(\"{rows:?}\");\n"; + +export const RsExampleTail = " Ok(())\n}\n"; + diff --git a/docs/web/snippets/tables.mdx b/docs/web/snippets/tables.mdx index 6b9b98280..28b9ce172 100644 --- a/docs/web/snippets/tables.mdx +++ b/docs/web/snippets/tables.mdx @@ -90,7 +90,7 @@ export const PyMergeUpdateInsert = "import pyarrow as pa\n\ntable = db.create_ta export const PyOpenExistingTable = "db = tmp_db\n# Create a table first\ndata = [{\"vector\": [1.1, 1.2], \"lat\": 45.5, \"long\": -122.7}]\ndb.create_table(\"test_table\", data, mode=\"overwrite\")\n\n# List table names\nprint(db.list_tables().tables)\n\n# Open existing table\ntbl = db.open_table(\"test_table\")\n"; -export const PySchemaAddSetup = "table_name = \"schema_evolution_add_example\"\nif data is None:\n data = [\n {\n \"id\": 1,\n \"name\": \"Laptop\",\n \"price\": 1200.00,\n \"vector\": np.random.random(128).tolist(),\n },\n {\n \"id\": 2,\n \"name\": \"Smartphone\",\n \"price\": 800.00,\n \"vector\": np.random.random(128).tolist(),\n },\n {\n \"id\": 3,\n \"name\": \"Headphones\",\n \"price\": 150.00,\n \"vector\": np.random.random(128).tolist(),\n },\n ]\ntable = tmp_db.create_table(table_name, data, mode=\"overwrite\")\n"; +export const PySchemaAddSetup = "import numpy as np\n\ndata = [\n {\n \"id\": 1,\n \"name\": \"Laptop\",\n \"price\": 1200.00,\n \"vector\": np.random.random(128).tolist(),\n },\n {\n \"id\": 2,\n \"name\": \"Smartphone\",\n \"price\": 800.00,\n \"vector\": np.random.random(128).tolist(),\n },\n {\n \"id\": 3,\n \"name\": \"Headphones\",\n \"price\": 150.00,\n \"vector\": np.random.random(128).tolist(),\n },\n]\ntable = db.create_table(\"schema_evolution_add_example\", data, mode=\"overwrite\")\n"; export const PySchemaAlterSetup = "table_name = \"schema_evolution_alter_example\"\nif data is None:\n data = [\n {\n \"id\": 1,\n \"name\": \"Laptop\",\n \"price\": 1200,\n \"discount_price\": 1080.0,\n \"vector\": np.random.random(128).tolist(),\n },\n {\n \"id\": 2,\n \"name\": \"Smartphone\",\n \"price\": 800,\n \"discount_price\": 720.0,\n \"vector\": np.random.random(128).tolist(),\n },\n ]\nschema = pa.schema(\n {\n \"id\": pa.int64(),\n \"name\": pa.string(),\n \"price\": pa.int32(),\n \"discount_price\": pa.float64(),\n \"vector\": pa.list_(pa.float32(), 128),\n }\n)\ntable = tmp_db.create_table(table_name, data, schema=schema, mode=\"overwrite\")\n"; diff --git a/docs/web/sql/data-types.mdx b/docs/web/sql/data-types.mdx index 70bafdb96..8c7fa1781 100644 --- a/docs/web/sql/data-types.mdx +++ b/docs/web/sql/data-types.mdx @@ -6,7 +6,7 @@ icon: shapes keywords: ["sql types", "arrow", "cast", "pseudo-columns", "vector", "blob", "json"] --- -Enterprise [SQL runs on a deployment.](/sql) +Enterprise [SQL runs in LanceDB Enterprise.](/sql) There is no SQL type layer over Lance storage. The table provider hands DataFusion the dataset's Arrow schema directly, so a column is the same object @@ -77,7 +77,7 @@ SELECT id FROM documents WHERE vector IS NOT NULL A list representation does not imply that ordinary arithmetic operators perform vector arithmetic. Use supported distance functions for similarity -calculations. SQL vector search depends on the deployment version and query +calculations. SQL vector search depends on the Enterprise server version and query plan; see [SQL](/basics/sql#no-vector-sql). diff --git a/docs/web/sql/ddl/function.mdx b/docs/web/sql/ddl/function.mdx index 39a661fc3..f0782008d 100644 --- a/docs/web/sql/ddl/function.mdx +++ b/docs/web/sql/ddl/function.mdx @@ -6,17 +6,22 @@ icon: function keywords: ["create function", "drop function", "show functions", "sql", "udf"] --- -Enterprise [SQL runs on a deployment.](/sql) +Enterprise [SQL runs in LanceDB Enterprise.](/sql) ## CREATE FUNCTION {#create} ```sql -CREATE FUNCTION f(x BIGINT) RETURNS BIGINT +CREATE FUNCTION f(x int64) RETURNS int64 LANGUAGE python AS 'def f(x): return x * 2' WITH (pip = 'numpy==1.26.4', python_version = '3.11') ``` +Parameter and result types are Arrow type names, not SQL ones: `int64`, +`float64`, `utf8`, `bool`, `date32`, and lists such as `list` or +`fixed_size_list`. `BIGINT` is refused. A parameter accepts nulls +unless it is declared `NOT NULL`. + The body is Python source. Its dependencies are declared with it and are part of the **immutable function version** -- changing one produces a new version rather than altering the registered one, so a column filled by an earlier @@ -27,7 +32,7 @@ version is unaffected. | `pip` | pip requirements for the worker environment | | `conda` | conda packages instead of pip | | `conda_channels` | channels in priority order; requires `conda` | -| `python_version` | worker Python version, defaulting to yours | +| `python_version` | worker Python version; `3.12` when omitted, whatever Python you send the statement from | | `gpu` | `'true'` when every execution needs a GPU | **Errors and limits** diff --git a/docs/web/sql/ddl/index.mdx b/docs/web/sql/ddl/index.mdx index 2ac18dd4c..3181210c0 100644 --- a/docs/web/sql/ddl/index.mdx +++ b/docs/web/sql/ddl/index.mdx @@ -1,30 +1,38 @@ --- title: "Index" sidebarTitle: "Index" -description: "CREATE INDEX and DROP INDEX — one column, server-chosen name." +description: "CREATE INDEX and DROP INDEX — one column, with the algorithm chosen by USING." icon: bolt keywords: ["create index", "drop index", "sql"] --- -Enterprise [SQL runs on a deployment.](/sql) +Enterprise [SQL runs in LanceDB Enterprise.](/sql) ## CREATE INDEX {#create} ```sql -CREATE INDEX ON documents (title) +CREATE INDEX ON documents USING btree (title) +CREATE INDEX text_search ON documents USING fts (text) ``` +`USING` is required: it names the algorithm. The scalar algorithms are `btree`, +`bitmap` and `label_list`; `fts` builds a full-text index; and the vector +algorithms are `ivf_pq`, `ivf_rq`, `ivf_flat`, `ivf_hnsw_sq` and `ivf_hnsw_pq`, +where the deployment supports them. Index options go in a `WITH (...)` clause. + -SQL `CREATE INDEX` covers **one column** and takes **no user-chosen name** -- -the server names it. For a multi-column or explicitly-named index, or to -choose the algorithm, use the SDK's -[`create_index`](/indexing). +In LanceDB Enterprise, SQL `CREATE INDEX` covers **one column**. A name between +`INDEX` and `ON` names the index, where the deployment's configuration allows +it; without one, the server names it after the column, as in `title_idx`. For a +multi-column index, use the SDK's [`create_index`](/indexing). -**Enterprise job type:** `create_ivf_pq`, `create_btree`, `create_fts`, and one -more per algorithm -- the type names which algorithm is building, not a -different feature. See [Build & Repair](/indexing/reindexing). +**Enterprise job type:** `reindex_` followed by the algorithm, such as +`reindex_btree` or `reindex_ivf_pq`; the name says which algorithm is building, +not a different feature. An SDK build that must not replace an existing index +runs as `create_` followed by the algorithm instead. See +[Build & Repair](/indexing/reindexing). ## DROP INDEX {#drop} @@ -35,7 +43,9 @@ DROP INDEX documents.title_idx Takes the qualified `table.index` name. Find it with the SDK's [`list_indices`](/indexing) if you created the index without -naming it yourself. +naming it yourself. A deployment builds the index in the background after +`CREATE INDEX` returns; until that build finishes, the index is not listed and +`DROP INDEX` reports it as not found. ## What to read next {#next} diff --git a/docs/web/sql/ddl/job.mdx b/docs/web/sql/ddl/job.mdx index ae99b014e..bc985d34d 100644 --- a/docs/web/sql/ddl/job.mdx +++ b/docs/web/sql/ddl/job.mdx @@ -6,7 +6,7 @@ icon: hourglass-half keywords: ["show jobs", "show errors", "pause job", "resume job", "cancel job", "sql"] --- -Enterprise [SQL runs on a deployment.](/sql) +Enterprise [SQL runs in LanceDB Enterprise.](/sql) A [job](/basics/jobs) is not a catalog object -- it has no schema and is not namespaced -- but it has its own SQL statements, grouped here the same way the @@ -73,7 +73,7 @@ being ignored. Cancellation is a request, matching the SDK's [`cancel()`](/basics/jobs#managing): it does not guarantee the job stops before doing more work, and asking twice or cancelling a finished job does nothing. -`CANCEL JOB` itself can be unavailable on a deployment with no job registry +`CANCEL JOB` itself can be unavailable on an Enterprise cluster with no job registry configured, which is a deployment condition rather than something you can work around from SQL. diff --git a/docs/web/sql/ddl/materialized-view.mdx b/docs/web/sql/ddl/materialized-view.mdx index 51b5ef55b..19dba5aa4 100644 --- a/docs/web/sql/ddl/materialized-view.mdx +++ b/docs/web/sql/ddl/materialized-view.mdx @@ -6,13 +6,13 @@ icon: table keywords: ["materialized view", "create materialized view", "show materialized views", "sql"] --- -Enterprise [SQL runs on a deployment.](/sql) +Enterprise [SQL runs in LanceDB Enterprise.](/sql) A materialized view stores a query's result as rows, kept current by a refresh -- the counterpart to a [view](/sql/ddl/view), which stores only the query. -Materialized views are off by default on a deployment and have to be enabled +Materialized views are off by default in LanceDB Enterprise and have to be enabled there. Which statements and query shapes a deployment accepts, whether it needs the source table to have stable row ids, and how it refreshes a view depend on its LanceDB Enterprise version and configuration. @@ -24,10 +24,9 @@ its LanceDB Enterprise version and configuration. CREATE MATERIALIZED VIEW AS SELECT ... ``` -On a `db://` connection, the SDK's -[`create_materialized_view`](/management/materialized-view#create) sends this -statement. Once created, a view is queried like a table, refreshed with SQL and -listed with SQL. +Once created, a materialized view can be queried like a table. Use the +statements below to list, refresh and drop it. For SDK methods, see +[Materialized views](/management/materialized-view#create). ## SHOW MATERIALIZED VIEWS {#show} @@ -36,8 +35,8 @@ SHOW MATERIALIZED VIEWS ``` -Cheaper than the SDK equivalent. `list_materialized_views` finds views by -reading every table's schema, costing one open per table; this asks the server. +This asks the server for its views, as the SDK's `list_materialized_views` does +on a `db://` connection. ## Refreshing one {#refresh} @@ -47,7 +46,7 @@ REFRESH MATERIALIZED VIEW chunks ``` See [REFRESH](/sql/dml/refresh#refresh-materialized-view) for the full syntax, -including `CASCADE` and `full = true`. +and compatibility notes for older server versions. ## Dropping one {#drop} @@ -57,8 +56,8 @@ stored rows, unlike dropping a [view](/sql/ddl/view#drop). **Enterprise job type:** `mat_view_refresh` for a refresh, or `sql` on a -deployment that rebuilds views through SQL; `drop_table` for dropping one, same -as any table. +deployment that rebuilds views through SQL. Dropping one runs the same job as +dropping any table, `drop_table` or `drop_object`. ## What to read next {#next} @@ -68,6 +67,6 @@ as any table. Creating one from the SDK. - How a refresh works, locally and on a deployment. + How a refresh works, locally and in LanceDB Enterprise. diff --git a/docs/web/sql/ddl/namespace.mdx b/docs/web/sql/ddl/namespace.mdx index daedf9720..efd209b02 100644 --- a/docs/web/sql/ddl/namespace.mdx +++ b/docs/web/sql/ddl/namespace.mdx @@ -6,7 +6,7 @@ icon: folder-tree keywords: ["namespace", "schema", "create schema", "drop schema", "sql"] --- -Enterprise [SQL runs on a deployment.](/sql) +Enterprise [SQL runs in LanceDB Enterprise.](/sql) A [namespace](/management/namespace) is addressed in SQL as a `SCHEMA` -- the standard's name for the same concept. @@ -14,14 +14,16 @@ standard's name for the same concept. ## CREATE SCHEMA {#create} ```sql -CREATE SCHEMA public -CREATE SCHEMA db1.ns1.ns2 +CREATE SCHEMA analytics +CREATE SCHEMA db1.`ns1$ns2` CREATE SCHEMA IF NOT EXISTS `ns1$ns2` ``` -A dotted path creates the whole chain: `db1.ns1.ns2` creates `ns2` inside -`ns1` inside `db1`, provided the parents already exist. A name containing the -namespace path delimiter itself is backtick-quoted, as in `` `ns1$ns2` `` above. +A nested namespace is one backtick-quoted name with `$` between its +components: `` db1.`ns1$ns2` `` creates `ns2` inside `ns1` in database `db1`, +provided `ns1` already exists. The part before the dot is the database, so a +three-part path such as `db1.ns1.ns2` is rejected. `public` is the database's +root namespace, which is never created or dropped. `IF NOT EXISTS` makes the statement a no-op rather than an error when the namespace is already there. @@ -29,14 +31,14 @@ namespace is already there. ## DROP SCHEMA {#drop} ```sql -DROP SCHEMA db1.public +DROP SCHEMA db1.analytics DROP SCHEMA IF EXISTS `a$b` DROP SCHEMA `a$b` RESTRICT ``` `CASCADE` is rejected: **"DROP SCHEMA CASCADE is not supported; drop the -objects inside first."** A namespace's tables, views and functions must be +objects inside first."** A namespace's tables, views and other objects must be dropped explicitly before the namespace itself can go. `RESTRICT` -- the default -- states that requirement rather than changing it. diff --git a/docs/web/sql/ddl/table.mdx b/docs/web/sql/ddl/table.mdx index bd64458be..390c24b65 100644 --- a/docs/web/sql/ddl/table.mdx +++ b/docs/web/sql/ddl/table.mdx @@ -6,7 +6,7 @@ icon: table keywords: ["create table", "alter table", "drop table", "describe", "clone", "ddl"] --- -Enterprise [SQL runs on a deployment.](/sql) +Enterprise [SQL runs in LanceDB Enterprise.](/sql) ## CREATE TABLE {#create} @@ -30,16 +30,15 @@ The table's columns and types, without a plan -- the SQL counterpart of ## ALTER TABLE {#alter} -`ALTER TABLE ... ADD COLUMN` covers three different things, distinguished by -what follows the column name. +What `ALTER TABLE ... ADD COLUMN` does depends on what follows the column +name. ### A plain column {#plain-column} -```sql -ALTER TABLE t ADD COLUMN c INT -``` - -Added empty and nullable. Nothing is computed. +SQL `ADD COLUMN` requires an `AS` binding, so a plain column such as +`ALTER TABLE t ADD COLUMN c INT` is rejected. Add an empty, nullable column +with the SDK instead -- see +[Add nullable columns](/management/table#schema-add-nullable-columns). ### A computed column {#computed-column} @@ -52,6 +51,9 @@ The column is committed empty and the expression is stored, not evaluated -- declaring one on a billion-row table costs what declaring one on an empty table costs. [`REFRESH COLUMN`](/sql/dml/refresh) fills it. +{/* +Draft for the Functions release. + ### A function column {#function-column} ```sql @@ -86,6 +88,7 @@ ALTER TABLE t ADD COLUMN c AS f(x) WITH (num_workers = '2') `WITH` on a plain computed column accepts only `version`. Resource options such as `num_workers` apply to function columns, where there are workers to size. +*/} ### Table properties {#properties} @@ -95,19 +98,23 @@ ALTER TABLE t SET UNENFORCED PRIMARY KEY (id) Records which column identifies a row. **Unenforced** is not a hedge in the naming: uniqueness is not checked on write, and declaring it does not make -duplicates impossible. It tells the planner what to assume. +duplicates impossible. It tells the planner what to assume. The column must be +declared `NOT NULL`. ```sql ALTER TABLE t SET WAL -ALTER TABLE t SET WAL WITH (sharding = 'bucket', buckets = 4) +ALTER TABLE t SET WAL WITH (sharding = 'bucket', column = 'id', buckets = 4) ALTER TABLE t SET WAL WITH (maintained_indexes = '') ALTER TABLE t UNSET WAL ``` A write-ahead log makes small frequent writes cheap by batching them out of the commit path. `sharding` and `buckets` spread the log so concurrent writers do -not contend on one place; `maintained_indexes` chooses which indexes are kept -current against WAL rows. See +not contend on one place; `bucket` sharding names its `column`, which must be +the table's unenforced primary key, and inserts into the table then need a +scalar index on that key. `maintained_indexes` chooses which indexes are kept +current against WAL rows. A table with a computed column cannot take a +write-ahead log. See [Background jobs](/maintenance/background-jobs#wal) for how the log gets folded back into the table. @@ -162,7 +169,8 @@ Removes the table and every version of it. There is no undrop. -**Enterprise job type:** `drop_table`. See +**Enterprise job type:** `drop_table` or `drop_object`, depending on how the +deployment stores its catalog. See [Background jobs](/maintenance/background-jobs#teardown). diff --git a/docs/web/sql/ddl/view.mdx b/docs/web/sql/ddl/view.mdx index fc0807058..8b4337fac 100644 --- a/docs/web/sql/ddl/view.mdx +++ b/docs/web/sql/ddl/view.mdx @@ -6,7 +6,7 @@ icon: eye keywords: ["create view", "drop view", "sql", "catalog"] --- -Enterprise [SQL runs on a deployment.](/sql) +Enterprise [SQL runs in LanceDB Enterprise.](/sql) A view stores a **query**, not rows. Reading it runs the query against current data, so a view is never stale -- and never faster than the query it wraps. It diff --git a/docs/web/sql/dml/copy.mdx b/docs/web/sql/dml/copy.mdx index 837b92285..658b20b59 100644 --- a/docs/web/sql/dml/copy.mdx +++ b/docs/web/sql/dml/copy.mdx @@ -6,7 +6,11 @@ icon: file-export keywords: ["copy", "export", "parquet", "sql"] --- -Enterprise [SQL runs on a deployment.](/sql) + +Recent deployment versions refuse `COPY` over the SQL endpoint with "COPY is not +supported". On those, export a result by reading it through a Flight SQL client +or an SDK and writing the files from your application. + ```sql COPY (SELECT id, title FROM documents) TO 's3://bucket/export/' STORED AS PARQUET diff --git a/docs/web/sql/dml/delete.mdx b/docs/web/sql/dml/delete.mdx index da71dd9eb..90965062b 100644 --- a/docs/web/sql/dml/delete.mdx +++ b/docs/web/sql/dml/delete.mdx @@ -6,12 +6,16 @@ icon: trash keywords: ["delete", "sql"] --- -Enterprise [SQL runs on a deployment.](/sql) +Enterprise [SQL runs in LanceDB Enterprise.](/sql) ```sql -DELETE FROM documents WHERE created_at < '2025-01-01' +DELETE FROM documents WHERE created_at < TIMESTAMP '2025-01-01 00:00:00' ``` +Compare a timestamp column with a typed literal, as above, or with +`CAST('2025-01-01' AS TIMESTAMP)`. A bare string such as `'2025-01-01'` is +rejected in a `DELETE` predicate, even though a `SELECT` accepts it. + Deletion marks rows rather than erasing them, so the data remains recoverable for as long as the version holding it survives [retention](/maintenance/cleanup). diff --git a/docs/web/sql/dml/insert.mdx b/docs/web/sql/dml/insert.mdx index 2be017f06..61fcb9994 100644 --- a/docs/web/sql/dml/insert.mdx +++ b/docs/web/sql/dml/insert.mdx @@ -6,7 +6,7 @@ icon: plus keywords: ["insert", "insert into", "sql"] --- -Enterprise [SQL runs on a deployment.](/sql) +Enterprise [SQL runs in LanceDB Enterprise.](/sql) ```sql INSERT INTO documents (id, title) VALUES (1, 'Merlin'), (2, 'Arthur') diff --git a/docs/web/sql/dml/refresh.mdx b/docs/web/sql/dml/refresh.mdx index 2d00c1104..1a14adef8 100644 --- a/docs/web/sql/dml/refresh.mdx +++ b/docs/web/sql/dml/refresh.mdx @@ -6,7 +6,7 @@ icon: arrows-rotate keywords: ["refresh column", "refresh materialized view", "backfill", "cascade"] --- -Enterprise [SQL runs on a deployment.](/sql) +Enterprise [SQL runs in LanceDB Enterprise.](/sql) LanceDB extensions with no DataFusion equivalent. They are what makes [feature engineering](/feature-engineering) reachable from SQL: a column @@ -52,33 +52,35 @@ REFRESH COLUMN embedding ON documents `num_workers` sets a count; the budget options size each worker. They are alternatives rather than a pair. +{/* +Draft for the Functions release. + `error_policy` and `num_workers` are refused for a column filled by a registered [function](/management/function) with its own resource budget -- use the CPU, memory and GPU options there. +*/} ## REFRESH MATERIALIZED VIEW {#refresh-materialized-view} ```sql -REFRESH MATERIALIZED VIEW [CASCADE] [WITH (