rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-13 15:22:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into refactor/wal-naming

Browse Source 
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Daniel Rammer
2026-10-12 15:30:00 -05:00
co-authored by Claude Opus 5.5
parent ab74aa620c 25fcfc696a
commit d5544cd5d9
636 changed files with 118151 additions and 22369 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.bumpversion.toml
+1 -1
View File
@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.38.0-beta.11"
current_version = "0.41.0-beta.6"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.
.github/dependabot.yml
+24
View File
@@ -44,3 +44,27 @@ updates:
python-deps:
patterns:
- "*"
# The npm ecosystem covers pnpm lockfiles. There are two separate installs:
# the bindings themselves and the examples, which have their own lockfile.
# As with cargo and pip above, only bump the lockfile — the version ranges
# in package.json are our consumers' constraints, not ours.
- package-ecosystem: npm
directory: /nodejs
schedule:
interval: weekly
versioning-strategy: lockfile-only
groups:
nodejs-deps:
patterns:
- "*"
- package-ecosystem: npm
directory: /nodejs/examples
schedule:
interval: weekly
versioning-strategy: lockfile-only
groups:
nodejs-examples-deps:
patterns:
- "*"
.github/release_notes.json
+5 -1
View File
@@ -6,6 +6,10 @@
"title": "## 🏆 Highlights",
"labels": ["highlight"]
},
{
"title": "## 🚨 Critical Fixes",
"labels": ["critical-fix"]
},
{
"title": "## 🛠 Breaking Changes",
"labels": ["breaking-change"]
@@ -38,4 +42,4 @@
"labels": ["ci"]
}
]
}
}
.github/workflows/dev.yml
+10 -4
View File
@@ -29,12 +29,14 @@ jobs:
steps:
- uses: actions/setup-node@v6
with:
node-version: "18"
node-version: "24"
- uses: pnpm/action-setup@v6
with:
version: 11.1.1
# These rules are disabled because Github will always ensure there
# is a blank line between the title and the body and Github will
# word wrap the description field to ensure a reasonable max line
# length.
- run: npm install @commitlint/config-conventional
- run: >
echo 'module.exports = {
"rules": {
@@ -43,7 +45,11 @@ jobs:
"body-leading-blank": [0, "always"]
}
}' > .commitlintrc.js
- run: npx commitlint --extends @commitlint/config-conventional --verbose <<< $COMMIT_MSG
- run: >
pnpm dlx
--package @commitlint/cli@21.2.2
--package @commitlint/config-conventional@21.2.2
commitlint --extends @commitlint/config-conventional --verbose <<< $COMMIT_MSG
env:
COMMIT_MSG: >
${{ github.event.pull_request.title }}
@@ -54,7 +60,7 @@ jobs:
with:
script: |
const message = `**ACTION NEEDED**
Lance follows the [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0/) for release automation.
The PR title and description are used as the merge commit message.\
.github/workflows/docs-link-check.yml
+1 -1
View File
@@ -56,7 +56,7 @@ jobs:
uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2.9.0
with:
# Restricted to http(s) on purpose. Much of docs/src is generated
# API reference (the js/ tree comes from `npm run docs` in nodejs)
# API reference (the js/ tree comes from `pnpm run docs` in nodejs)
# and the hand-written pages use mkdocstrings cross-references and
# nav-relative paths that only resolve in the site mkdocs builds,
# not in this checkout, so relative links would be reported as
.github/workflows/docs.yml
+1 -3
View File
@@ -55,9 +55,7 @@ jobs:
- name: Set up node
uses: actions/setup-node@v6
with:
node-version: 20
cache: 'npm'
cache-dependency-path: docs/package-lock.json
node-version: 24
- name: Install node dependencies
working-directory: nodejs
run: |
.github/workflows/license-header-check.yml
+1
View File
@@ -9,6 +9,7 @@ on:
- python/**
- nodejs/**
- java/**
- docs/web-tests/**
- .github/workflows/license-header-check.yml
permissions:
.github/workflows/nodejs.yml
+77 -14
View File
@@ -47,9 +47,8 @@ jobs:
version: 11.1.1
- uses: actions/setup-node@v6
with:
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
# in October. The library itself still supports Node >= 18
# (see test matrix below).
# Build on a supported LTS; the matrix job below covers every
# Node version the library claims to support.
node-version: 24
cache: 'pnpm'
cache-dependency-path: nodejs/pnpm-lock.yaml
@@ -84,7 +83,7 @@ jobs:
timeout-minutes: 30
strategy:
matrix:
node-version: [ "18", "20" ]
node-version: [ "22", "24", "26" ]
runs-on: "ubuntu-22.04"
defaults:
run:
@@ -101,9 +100,9 @@ jobs:
- uses: actions/setup-node@v6
name: Setup Node.js 24 for build
with:
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
# in October. Build/install runs on Node 24; tests run on the
# matrix version below using direct jest invocation.
# Build and install once on a fixed version so the generated docs
# are identical across matrix legs; the tests below then run on each
# supported Node version.
node-version: 24
cache: 'pnpm'
cache-dependency-path: nodejs/pnpm-lock.yaml
@@ -152,9 +151,9 @@ jobs:
S3_TEST: "1"
# Newer @smithy/core uses dynamic ESM imports.
NODE_OPTIONS: "--experimental-vm-modules"
# Invoke jest directly because pnpm 11 itself requires Node 22+
# while the matrix tests on older Node versions.
run: npx jest --verbose
# Invoke the installed jest binary directly; the pnpm shim is set up
# against the build-phase Node, not the version selected above.
run: node_modules/.bin/jest --verbose
- name: Test examples
working-directory: ./
env:
@@ -164,7 +163,72 @@ jobs:
run: |
python ci/mock_openai.py &
cd nodejs/examples
npx jest --testEnvironment jest-environment-node-single-context --verbose
node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose
arrow-compat:
# The main jobs resolve `apache-arrow` to the version pinned in
# devDependencies (the newest supported release). This job re-runs the
# suite with the library itself running on each older Arrow release the
# peer range still accepts, which is what a consumer on that version gets.
name: Linux (Arrow ${{ matrix.arrow-version }})
timeout-minutes: 30
strategy:
matrix:
arrow-version: [ "19.0.1", "20.0.0" ]
runs-on: "ubuntu-22.04"
defaults:
run:
shell: bash
working-directory: nodejs
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
lfs: true
- uses: pnpm/action-setup@v6
with:
version: 11.1.1
- uses: actions/setup-node@v6
with:
node-version: 24
cache: 'pnpm'
cache-dependency-path: nodejs/pnpm-lock.yaml
- uses: Swatinem/rust-cache@v2
with:
# Same crate, profile and target as the `linux` job above, so reuse
# its cache: rust-cache keys on the job id by default, and this job
# would otherwise start cold on every run until it had saved its own.
shared-key: linux
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Install dependencies
run: |
sudo apt update
sudo apt install -y protobuf-compiler libssl-dev
- name: Build
run: |
pnpm install --frozen-lockfile
# No `--` separator: pnpm forwards it literally, which would
# make napi-rs treat `--profile ci` as a cargo passthrough arg.
pnpm build:debug --profile ci
- name: Resolve apache-arrow to ${{ matrix.arrow-version }}
run: |
# The older releases are already installed as the `apache-arrow-NN`
# test aliases. With pnpm's hoisted linker each alias is a complete,
# self-contained directory, so swap it in for the library's own
# `apache-arrow` instead of running a second install, which has been
# seen to hang after printing "Done" in CI. A copy, not a symlink: the
# cross-version tests rely on the alias being a separate module
# instance from the one the library loads.
major="$(echo '${{ matrix.arrow-version }}' | cut -d. -f1)"
mv node_modules/apache-arrow node_modules/apache-arrow.pinned
cp -R "node_modules/apache-arrow-${major}" node_modules/apache-arrow
# `require("apache-arrow/package.json")` is blocked by the package's exports map.
node -p "'apache-arrow resolved to ' + JSON.parse(require('fs').readFileSync('node_modules/apache-arrow/package.json')).version"
pnpm tsc
- name: Test
env:
# Newer @smithy/core uses dynamic ESM imports.
NODE_OPTIONS: "--experimental-vm-modules"
run: node_modules/.bin/jest --verbose
macos:
timeout-minutes: 30
# macos-15 ships a newer linker; the older macos-14 linker fails to insert
@@ -185,8 +249,7 @@ jobs:
version: 11.1.1
- uses: actions/setup-node@v6
with:
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
# in October.
# pnpm 11 requires Node >= 22.13.
node-version: 24
cache: 'pnpm'
cache-dependency-path: nodejs/pnpm-lock.yaml
@@ -209,4 +272,4 @@ jobs:
pnpm tsc
- name: Test
run: |
pnpm test
pnpm test --runInBand
.github/workflows/npm-publish.yml
+81 -36
View File
@@ -40,40 +40,31 @@ jobs:
- target: aarch64-apple-darwin
host: macos-latest
features: fp16kernels
# Fat LTO was ~111 of this job's ~113 minutes.
lto: thin
codegen_units: 16
pre_build: |-
brew install protobuf
# Fat LTO (the workspace default in .cargo/config.toml) is
# single-threaded and is the peak-memory step of the build. On
# this runner it accounted for ~111 of the job's ~113 minutes,
# making it the critical path of the entire publish pipeline.
# ThinLTO parallelizes it across the runner's cores, for a few
# percent of runtime performance.
export CARGO_PROFILE_RELEASE_LTO=thin
export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
- target: x86_64-pc-windows-msvc
host: windows-2025
features: ","
# The lower peak also keeps this on the standard 4-core runner.
lto: thin
codegen_units: 16
pre_build: |-
choco install --no-progress protoc ninja nasm
tail -n 1000 /c/ProgramData/chocolatey/logs/chocolatey.log
# There is an issue where choco doesn't add nasm to the path
export PATH="$PATH:/c/Program Files/NASM"
nasm -v
# See the ThinLTO note on aarch64-apple-darwin above. Keeping
# peak memory down is also what lets this run on the standard
# 4-core runner: the 8-core larger runner was only needed to
# stop fat LTO from OOMing rustc-LLVM.
export CARGO_PROFILE_RELEASE_LTO=thin
export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
- target: aarch64-pc-windows-msvc
host: windows-2025
features: ","
lto: thin
codegen_units: 16
pre_build: |-
choco install --no-progress protoc
rustup target add aarch64-pc-windows-msvc
# See the ThinLTO note on aarch64-apple-darwin above.
export CARGO_PROFILE_RELEASE_LTO=thin
export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
- target: x86_64-unknown-linux-gnu
host: ubuntu-latest
features: fp16kernels
@@ -103,6 +94,14 @@ jobs:
# https://github.com/napi-rs/napi-rs/blob/main/debian-aarch64.Dockerfile
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian-aarch64
features: "fp16kernels"
# Fat LTO OOM-killed rustc every nightly; even with lld it peaked
# at 31391 MiB of the runner's 32 GiB.
lto: thin
codegen_units: 16
# arm64 Linux links through GNU `ld` where x86_64 defaults to
# `rust-lld`, which is why only arm64 OOM'd. lld cut the largest
# linker process 7.0 -> 4.0 GiB (lancedb/sophon#7313).
linker: /tmp/aarch64-lld-clang
pre_build: |-
set -e &&
apt-get update &&
@@ -112,9 +111,30 @@ jobs:
# AT_HWCAP2 (added in Linux 3.17). Define it for aws-lc-sys.
export CFLAGS="$CFLAGS -DAT_HWCAP2=26" &&
rustup target add aarch64-unknown-linux-gnu
# Not `&&`-chained: in dash, errexit does not fire for a
# non-final command in an `&&` list, so failures were ignored.
#
# A wrapper rather than `-C link-arg` because the per-target
# rustflags variable does not reach every unit that links, while
# the linker variable does. `clang` because GCC silently ignores
# `-fuse-ld=lld` unless built with lld support. Two echoes
# because printf's newline escape gets rewritten to `;` between
# here and the container.
echo '#!/bin/sh' > /tmp/aarch64-lld-clang
echo 'exec clang --target=aarch64-unknown-linux-gnu --sysroot=/usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot --gcc-toolchain=/usr/aarch64-unknown-linux-gnu -fuse-ld=lld "$@"' >> /tmp/aarch64-lld-clang
chmod 0755 /tmp/aarch64-lld-clang
# Fail now, not at the cdylib link ~30 minutes later. Linking at
# all also proves lld resolved; clang errors out when it cannot.
echo 'int main(void){return 0;}' > /tmp/probe.c
/tmp/aarch64-lld-clang /tmp/probe.c -o /tmp/probe
readelf -h /tmp/probe | grep AArch64
- target: aarch64-unknown-linux-musl
host: ubuntu-2404-8x-x64
features: ","
# Fat LTO took the whole runner down. lld cannot help: it died
# inside rustc's LLVM, before any linker was spawned.
lto: thin
codegen_units: 16
pre_build: |-
set -e &&
sudo apt-get update &&
@@ -123,6 +143,19 @@ jobs:
export EXTRA_ARGS="-x"
name: build - ${{ matrix.settings.target }}
runs-on: ${{ matrix.settings.host }}
# On the job, not exported from `pre_build`: `Swatinem/rust-cache` hashes
# `CARGO_*` into its cache key before any step runs, so a step-local export
# leaves the key unchanged while cargo still rebuilds cold. The ThinLTO
# legs had been doing that every run.
#
# Not `RUSTFLAGS`: setting it, even to "", discards every config-file
# rustflag, silently dropping .cargo/config.toml's `target-cpu` and
# `target-feature` from the published binaries.
env:
CARGO_PROFILE_RELEASE_LTO: ${{ matrix.settings.lto || 'fat' }}
CARGO_PROFILE_RELEASE_CODEGEN_UNITS: ${{ matrix.settings.codegen_units || '1' }}
# Empty elsewhere: a per-target variable is only read for that triple.
CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: ${{ matrix.settings.linker }}
defaults:
run:
working-directory: nodejs
@@ -135,8 +168,7 @@ jobs:
- name: Setup node
uses: actions/setup-node@v6
with:
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
# in October.
# pnpm 11 requires Node >= 22.13.
node-version: 24
cache: pnpm
cache-dependency-path: nodejs/pnpm-lock.yaml
@@ -169,19 +201,15 @@ jobs:
# creating ref). The nightly cadence also keeps entries inside
# GitHub's 7-day eviction window, which a tag-only trigger would not.
save-if: ${{ github.ref == 'refs/heads/main' }}
# Docker builds can use rust-cache too. `target/` already lives on the
# host because the whole workspace is bind-mounted into the container, and
# rust-cache's prune and save run host-side, so they can manage it -- which
# is what keeps the entry to dependency artifacts rather than a multi-GB
# copy of everything.
# Docker builds can use rust-cache too: the workspace is bind-mounted, so
# `target/` lives on the host and rust-cache's prune keeps the entry
# small.
#
# Two differences from the native builds. The container's CARGO_HOME is
# bind-mounted from `.cargo-cache` rather than the host's ~/.cargo, so that
# has to be cached explicitly. And the key is derived from the *host* rustc
# version, which is not the compiler that produced these artifacts; that is
# safe because cargo fingerprints the real compiler and rebuilds on a
# mismatch, it just means a base-image toolchain bump costs one cold build
# instead of invalidating the key.
# bind-mounted from `.cargo-cache` rather than ~/.cargo, so that is cached
# explicitly. And the key uses the *host* rustc version, not the compiler
# that built these artifacts -- safe, since cargo fingerprints the real
# one; a base-image bump just costs one cold build.
- name: Cache cargo (docker builds)
uses: Swatinem/rust-cache@v2
if: ${{ matrix.settings.docker }}
@@ -210,14 +238,19 @@ jobs:
# cache step above saves. Previously the registry mounts pointed at
# `.cargo/...`, a path nothing cached, so the container re-downloaded
# the whole crate registry on every run.
#
# `docker run` inherits nothing; `-e NAME` carries the job's `env:` in.
options: "--user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db \
-v ${{ github.workspace }}/.cargo-cache/registry/cache:/usr/local/cargo/registry/cache \
-v ${{ github.workspace }}/.cargo-cache/registry/index:/usr/local/cargo/registry/index \
-e CARGO_PROFILE_RELEASE_LTO \
-e CARGO_PROFILE_RELEASE_CODEGEN_UNITS \
-e CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER \
-v ${{ github.workspace }}:/build -w /build/nodejs"
run: |
set -e
${{ matrix.settings.pre_build }}
npx napi build --platform --release \
node_modules/.bin/napi build --platform --release \
--features ${{ matrix.settings.features }} \
--target ${{ matrix.settings.target }} \
--dts ../lancedb/native.d.ts \
@@ -237,7 +270,7 @@ jobs:
- name: Build
run: |
${{ matrix.settings.pre_build }}
npx napi build --platform --release \
node_modules/.bin/napi build --platform --release \
--features ${{ matrix.settings.features }} \
--target ${{ matrix.settings.target }} \
--dts ../lancedb/native.d.ts \
@@ -256,6 +289,18 @@ jobs:
if: always()
run: df -h
shell: bash
- name: Report peak memory
if: always() && runner.os == 'Linux'
shell: bash
run: |
peak=$(find /sys/fs/cgroup -name memory.peak -readable \
-exec cat {} + 2>/dev/null | sort -n | tail -1)
if [ -n "$peak" ]; then
echo "peak memory: $((peak / 1024 / 1024)) MiB"
else
echo "peak memory: unavailable (no readable cgroup v2 memory.peak)"
fi
free -g || true
- name: Upload artifact
uses: actions/upload-artifact@v7
with:
@@ -293,7 +338,7 @@ jobs:
- target: aarch64-unknown-linux-gnu
host: ubuntu-2404-8x-arm64
node:
- '20'
- '22'
runs-on: ${{ matrix.settings.host }}
defaults:
run:
@@ -339,9 +384,9 @@ jobs:
- name: Move built files
run: cp dist/native.d.ts dist/native.js dist/*.node lancedb/
- name: Test bindings
# Invoke jest directly because pnpm 11 itself requires Node 22+
# while the matrix tests on older Node versions.
run: npx jest --verbose
# Invoke the installed jest binary directly; the pnpm shim is set up
# against the install-phase Node, not the version selected above.
run: node_modules/.bin/jest --verbose
publish:
name: Publish
runs-on: ubuntu-latest
.github/workflows/pypi-publish.yml
+9 -10
View File
@@ -21,7 +21,7 @@ permissions:
contents: read
# Without this, a force-push to a PR leaves the previous run going -- including
# a ~74 minute Windows job and a billed arm64 wheel build.
# a long fat-LTO Windows job and a billed arm64 wheel build.
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
@@ -123,18 +123,17 @@ jobs:
path: target/wheels/lancedb-*.whl
if-no-files-found: error
windows:
timeout-minutes: 90
runs-on: windows-latest
# The fat-LTO build of the `lancedb` crate does not finish within 90 minutes
# on the 4-core standard runner, so use the 8-core larger runner with extra
# headroom.
timeout-minutes: 150
runs-on: windows-2025-8x-x64
env:
# link.exe is single-threaded and the long pole on Windows builds. Use
# rustc's bundled lld-link instead.
CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER: rust-lld
# Fat LTO of the cdylib is single-threaded and the peak-memory step of the
# build. ThinLTO parallelizes it across the runner's cores, at some cost
# to runtime performance on our least performance-sensitive platform.
# Matches what the nodejs Windows builds already do in npm-publish.yml.
CARGO_PROFILE_RELEASE_LTO: thin
CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 16
# Keep the repository's fat LTO / 1 codegen unit release profile to leave
# headroom below PyPI's file-size limit for Windows wheels.
steps:
- uses: actions/checkout@v6
with:
@@ -149,7 +148,7 @@ jobs:
# or the default branch -- so with no run on main there is nothing that
# can populate an entry the release build would be allowed to read. Fixing
# this needs a main/nightly trigger (which would also catch wheel-build
# breakage before a release); the ~74 minutes here is otherwise dominated
# breakage before a release); the build time here is otherwise dominated
# by the fat-LTO link, which no cache avoids.
- uses: ./.github/workflows/build_windows_wheel
with:
.github/workflows/python.yml
+1 -1
View File
@@ -156,7 +156,7 @@ jobs:
- name: Test without pylance or pandas
run: |
pip uninstall -y pylance pandas
pytest -vv python/tests/test_table.py
pytest -vv python/tests/test_table.py python/tests/test_namespace_no_pylance.py
# Make sure wheels are not included in the Rust cache
- name: Delete wheels
run: rm -rf target/wheels
.github/workflows/rust.yml
+10 -37
View File
@@ -180,7 +180,8 @@ jobs:
env:
CC: clang-18
CXX: clang++-18
GH_TOKEN: ${{ secrets.SOPHON_READ_TOKEN }}
GH_TOKEN: ${{ github.token }}
SOPHON_READ_TOKEN: ${{ secrets.SOPHON_READ_TOKEN }}
steps:
- uses: actions/checkout@v6
with:
@@ -232,7 +233,10 @@ jobs:
ALL_FEATURES=`cargo metadata --format-version=1 --no-deps \
| jq -r '.packages[] | .features | keys | .[]' \
| grep -v s3-test | sort | uniq | paste -s -d "," -`
cargo test --profile ci --features $ALL_FEATURES --locked
# Run doctests before test binaries fill the runner disk. Examples are
# already built by the Linux job, so avoid retaining them here.
cargo test --profile ci --features $ALL_FEATURES --locked --doc
cargo test --profile ci --features $ALL_FEATURES --locked --lib --tests
windows:
strategy:
@@ -281,7 +285,7 @@ jobs:
runs-on: ubuntu-24.04
strategy:
matrix:
msrv: ["1.91.0"] # This should match up with rust-version in Cargo.toml
msrv: ["1.94.0"] # This should match up with rust-version in Cargo.toml
env:
# Need up-to-date compilers for kernels
CC: clang-18
@@ -304,40 +308,9 @@ jobs:
# unreadable outside their own branch anyway, since GitHub scopes
# caches to the creating ref.
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Downgrade dependencies
# These packages have newer requirements for MSRV
run: |
cargo update -p aws-sdk-bedrockruntime --precise 1.77.0
cargo update -p aws-sdk-dynamodb --precise 1.68.0
cargo update -p aws-config --precise 1.6.0
cargo update -p aws-sdk-kms --precise 1.63.0
cargo update -p aws-sdk-s3 --precise 1.79.0
cargo update -p aws-sdk-sso --precise 1.62.0
cargo update -p aws-sdk-ssooidc --precise 1.63.0
cargo update -p aws-sdk-sts --precise 1.63.0
# aws-runtime/sigv4/credential-types/types and the aws-smithy-*
# crates bumped their MSRV to 1.91.1 in late 2026; pin to the last
# 1.91.0-compatible versions. The order matters — each downgrade
# only succeeds once everything that still pins it at a higher
# version has itself been downgraded.
cargo update -p aws-runtime --precise 1.5.12
cargo update -p aws-types --precise 1.3.9
cargo update -p aws-sigv4 --precise 1.3.5
cargo update -p aws-credential-types --precise 1.2.8
# aws-smithy-checksums must stay at or above 0.63.13: OpenDAL's S3
# service needs crc-fast ~1.9, and older releases pin it to ~1.3.
cargo update -p aws-smithy-checksums --precise 0.63.13
cargo update -p aws-smithy-runtime --precise 1.9.3
cargo update -p aws-smithy-http --precise 0.62.6
cargo update -p aws-smithy-eventstream --precise 0.60.14
cargo update -p aws-smithy-http-client --precise 1.1.3
cargo update -p aws-smithy-observability --precise 0.1.4
cargo update -p aws-smithy-query --precise 0.60.8
cargo update -p aws-smithy-runtime-api --precise 1.9.3
cargo update -p aws-smithy-async --precise 1.2.7
cargo update -p aws-smithy-types --precise 1.3.6
cargo update -p aws-smithy-xml --precise 0.60.11
cargo update -p home --precise 0.5.9
- name: Downgrade dependencies
# aws-smithy-types 1.6.1+ requires rustc 1.94.1
run: cargo update -p aws-smithy-types --precise 1.5.0
- name: cargo +${{ matrix.msrv }} check
env:
RUSTUP_TOOLCHAIN: ${{ matrix.msrv }}
.github/workflows/typos.yml
+20
View File
@@ -0,0 +1,20 @@
name: Typo checker
on:
push:
branches:
- main
pull_request:
permissions:
contents: read
jobs:
run:
name: Spell Check with Typos
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Check spelling of the entire repository
uses: crate-ci/typos@6802cc60d4e7f78b9d5454f6cf3935c042d5e1e3 # v1.26.0
.github/workflows/update_package_lock_run.yml
-22
View File
@@ -1,22 +0,0 @@
name: Update package-lock.json
on:
workflow_dispatch:
permissions:
contents: read
jobs:
publish:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
with:
ref: main
persist-credentials: false
fetch-depth: 0
lfs: true
- uses: ./.github/workflows/update_package_lock
with:
github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
.github/workflows/update_package_lock_run_nodejs.yml
-22
View File
@@ -1,22 +0,0 @@
name: Update NodeJs package-lock.json
on:
workflow_dispatch:
permissions:
contents: read
jobs:
publish:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
with:
ref: main
persist-credentials: false
fetch-depth: 0
lfs: true
- uses: ./.github/workflows/update_package_lock_nodejs
with:
github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
.pre-commit-config.yaml
+8 -1
View File
@@ -10,6 +10,10 @@ repos:
rev: v0.9.9
hooks:
- id: ruff
- repo: https://github.com/crate-ci/typos
rev: v1.26.0
hooks:
- id: typos
# - repo: https://github.com/RobertCraigie/pyright-python
# rev: v1.1.395
# hooks:
@@ -20,7 +24,10 @@ repos:
hooks:
- id: local-biome-check
name: biome check
entry: npx @biomejs/biome@1.8.3 check --config-path nodejs/biome.json nodejs/
# Use the biome from nodejs/package.json rather than a separately
# pinned one: the two drifted apart and disagreed on formatting, so
# this hook rejected code that `pnpm lint` accepted.
entry: nodejs/node_modules/.bin/biome check --config-path nodejs/biome.json nodejs/
language: system
types: [text]
files: "nodejs/.*"
.typos.toml
+33
View File
@@ -0,0 +1,33 @@
[default]
extend-ignore-re = [
"(?Rm)^.*(#|//)\\s*spellchecker:disable-line$",
# Conventional import aliases in the docs examples: `import fiftyone as fo`
# and `import torchvision.io as tio`.
"\\bas (fo|tio)\\b",
"\\btio\\.",
# The substring query in the n-gram full-text search example.
'search\("nce"',
# Garbled OCR output quoted verbatim in the VLM fine-tuning tutorial.
"permission nstallation",
]
[default.extend-words]
# Azure Kubernetes Service, mentioned in rust/lancedb/src/remote/oauth.rs.
AKS = "AKS"
# RabitQ is the name of a vector quantization algorithm, not a typo of "Rabbit".
Rabit = "Rabit"
# `VarBuilder::from_mmaped_safetensors` is the real (if oddly-spelled) name of
# the candle-core API we call in rust/lancedb/src/embeddings/sentence_transformers.rs.
mmaped = "mmaped"
# `WriteableBuffer` is the real name of a type from Python's `_typeshed` stubs,
# used in python/python/lancedb/_blob.py.
Writeable = "Writeable"
# Superlinked's self-hosted inference engine (SIE), its `sie-lancedb` package,
# and the `sie` embedding function it registers, in
# docs/web/integrations/embedding/superlinked.mdx.
SIE = "SIE"
[files]
extend-exclude = [
"*_THIRD_PARTY_LICENSES.*",
]
AGENTS.md
+3 -3
View File
@@ -38,7 +38,7 @@ Before committing changes, run formatting for every language you touched. At min
* Rust changes: run `cargo fmt --all`.
* Python changes: run `ruff format .` and `ruff check .` from the repository root,
and run targeted tests through `cd python && uv run ...`.
* TypeScript changes: run the relevant `npm`/`pnpm` lint, format, build, and docs commands in `nodejs`.
* TypeScript changes: run the relevant `pnpm` lint, format, build, and docs commands in `nodejs`.
Before creating a PR, the exact value passed to `gh pr create --title` must follow
Conventional Commits, such as `fix: support nested field paths in native index creation`
@@ -101,12 +101,12 @@ Python bindings changes:
TypeScript bindings changes:
1. Add napi-rs method binding on `Table` in `nodejs/src/table.rs`.
2. Run `npm run build` to generate TypeScript definitions.
2. Run `pnpm build` to generate TypeScript definitions.
3. Add typescript method on abstract class `Table` in `nodejs/src/table.ts`.
4. Add concrete method on `LocalTable` class in `nodejs/src/native_table.ts`.
* Note: despite the name, this class is also used for remote tables.
5. Add test in `nodejs/__test__/table.test.ts`.
6. Run `npm run docs` to generate TypeScript documentation.
6. Run `pnpm run docs` to generate TypeScript documentation.
## Python API reference
CLAUDE.md
-1
View File
@@ -1 +0,0 @@
AGENTS.md
CONTRIBUTING.md
+44 -1
View File
@@ -71,7 +71,50 @@ are wrappers around the Rust library.
* `nodejs`: Typescript package source code
* `node`: **Deprecated** Typescript package source code
* `java`: Java package source code
* `docs`: Documentation source code
* `docs/src`: SDK reference, built with mkdocs
* `docs/web`: the open-source pages of [docs.lancedb.com](https://docs.lancedb.com)
* `docs/web-tests`: the tests the documentation's code examples are extracted from
## Documentation
The open-source pages of [docs.lancedb.com](https://docs.lancedb.com) live in
`docs/web`, so a change to behaviour and the change to the page describing it
belong in the same pull request.
Preview them locally — this serves `docs/web` on its own, with no other checkout:
```bash
npm i -g mint # https://mintlify.com/docs/installation
cd docs/web && mint dev
```
This repository holds a page at every path the site publishes, so the whole
navigation resolves locally. Enterprise pages exist here as open-source pages --
what the capability is, and what the embedded form does instead. A separate
private repository supplies the fuller version of each, which replaces the page
at the same path when the published site is assembled.
Two things to know before editing:
**Code examples are not written into the pages.** They live in real tests under
`docs/web-tests/{py,ts,rs}`, are extracted into `docs/web/snippets/`, and are
imported by the pages. Edit the test, then regenerate from the repository root:
```bash
uv run docs/web-tests/mdx_snippets_gen.py -s docs/web-tests/py -s docs/web-tests/ts -s docs/web-tests/rs -o docs/web/snippets
```
Review the diff before committing: a few modules currently regenerate with
unrelated changes, so commit only the snippets for the examples you changed.
Run the Python, TypeScript and Rust examples in `docs/web-tests/{py,ts,rs}`
locally before submitting the change. These examples are not yet wired into
pull-request CI; that integration remains follow-up work.
**Every heading carries an explicit `{#anchor}`.** Those anchors are how the
Enterprise pages attach their additions to the right section, so they are
assigned once and never regenerated. Leave an existing anchor alone even when
you reword the heading above it; only new headings need a new one.
## Release process
Cargo.lock
Generated
+997 -517
View File
File diff suppressed because it is too large Load Diff
Cargo.toml
+22 -17
View File
@@ -10,23 +10,23 @@ repository = "https://github.com/lancedb/lancedb"
description = "Serverless, low-latency vector database for AI applications"
keywords = ["lancedb", "lance", "database", "vector", "search"]
categories = ["database-implementations"]
rust-version = "1.91.0"
rust-version = "1.94.0"
[workspace.dependencies]
lance = { "version" = "=12.0.0-beta.2", default-features = false, "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=12.0.0-beta.2", default-features = false, "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=12.0.0-beta.2", default-features = false, "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=14.0.0-beta.13", default-features = false, "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=14.0.0-beta.13", default-features = false, "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=14.0.0-beta.13", default-features = false, "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lancedb = { path = "rust/lancedb", default-features = false }
ahash = "0.8"
# Note that this one does not include pyarrow
@@ -39,7 +39,10 @@ arrow-ord = "58.0.0"
arrow-schema = "58.0.0"
arrow-select = "58.0.0"
arrow-cast = "58.0.0"
arrow-flight = { version = "58.0.0", features = ["flight-sql-experimental"] }
async-trait = "0"
# Smithy JSON 0.63 requires the pre-1.7 Document representation; allow the MSRV pin.
aws-smithy-types = ">=1.3.6, <1.7"
bytes = "1"
datafusion = { version = "54.0.0", default-features = false }
datafusion-catalog = "54.0.0"
@@ -59,19 +62,21 @@ log = "0.4"
metrics = "0.24"
metrics-util = "0.19"
moka = { version = "0.12", features = ["future"] }
object_store = "0.13.2"
object_store = "0.14.1"
pin-project = "1.0.7"
rand = "0.9"
snafu = "0.8"
url = "2"
num-traits = "0.2"
oauth2 = { version = "5.0", default-features = false }
regex = "1.10"
semver = "1.0.25"
serde = "1"
serde_json = "1"
tempfile = "3.5.0"
tokio = { version = "1.23", features = ["rt-multi-thread", "sync"] }
uuid = { version = "1.7.0", features = ["v4"] }
tonic = { version = "0.14", features = ["tls-native-roots", "tls-aws-lc"] }
uuid = { version = "1.7.0", features = ["v4", "v7"] }
chrono = { version = "0.4", default-features = false, features = ["clock"] }
[profile.ci]
Makefile
+1 -1
View File
@@ -5,5 +5,5 @@ licenses:
cd python && cargo about generate ../about.hbs -o RUST_THIRD_PARTY_LICENSES.html -c ../about.toml
cd python && uv sync --all-extras && uv tool run pip-licenses --python .venv/bin/python --format=markdown --with-urls --output-file=PYTHON_THIRD_PARTY_LICENSES.md
cd nodejs && cargo about generate ../about.hbs -o RUST_THIRD_PARTY_LICENSES.html -c ../about.toml
cd nodejs && npx license-checker --markdown --out NODEJS_THIRD_PARTY_LICENSES.md
cd nodejs && pnpm dlx license-checker@25 --markdown --out NODEJS_THIRD_PARTY_LICENSES.md
cd java && ./mvnw license:aggregate-add-third-party -q
README.md
+2 -4
View File
@@ -1,8 +1,6 @@
<a href="https://cloud.lancedb.com" target="_blank">
<img src="https://github.com/user-attachments/assets/92dad0a2-2a37-4ce1-b783-0d1b4f30a00c" alt="LanceDB Cloud Public Beta" width="100%" style="max-width: 100%;">
</a>
<div align="center">
[![Reverie](docs/src/assets/reverie-banner.png)](https://www.reveriesummit.com/?utm_medium=social&utm_source=github&utm_campaign=reverie-2026)
[![LanceDB](docs/src/assets/hero-header.png)](https://lancedb.com)
[![Website](https://img.shields.io/badge/-Website-100000?style=for-the-badge&labelColor=645cfb&color=645cfb)](https://lancedb.com/)
[![Blog](https://img.shields.io/badge/Blog-100000?style=for-the-badge&labelColor=645cfb&color=645cfb)](https://blog.lancedb.com/)
@@ -13,7 +11,7 @@
<img src="docs/src/assets/lancedb.png" alt="LanceDB" width="50%">
# **The Multimodal AI Lakehouse**
# **The Multimodal Lakehouse for AI**
[**How to Install** ](#how-to-install) ✦ [**Detailed Documentation**](https://docs.lancedb.com) ✦ [**Tutorials and Recipes**](https://github.com/lancedb/vectordb-recipes/tree/main) ✦ [**Contributors**](#contributors)
ci/update_lockfiles.sh
+2 -6
View File
@@ -12,16 +12,12 @@ done
# This updates the lockfile without building
cargo metadata --quiet > /dev/null
pushd nodejs || exit 1
npm install --package-lock-only --silent
popd
if git diff --quiet --exit-code; then
echo "No lockfile changes to commit; skipping amend."
elif $AMEND; then
git add Cargo.lock nodejs/package-lock.json
git add Cargo.lock
git commit --amend --no-edit
else
git add Cargo.lock nodejs/package-lock.json
git add Cargo.lock
git commit -m "Update lockfiles"
fi
deny.toml
+1 -11
View File
@@ -131,18 +131,13 @@ allow = [
"BSD-3-Clause",
"ISC",
"Unicode-3.0",
"Unicode-DFS-2016",
"Zlib",
"CC0-1.0",
"MPL-2.0",
"BSL-1.0",
"OpenSSL",
# 0BSD ("BSD Zero Clause") is effectively public domain — no attribution
# required. Pulled in by `mock_instant`.
"0BSD",
# bzip2-1.0.6 is the permissive upstream bzip2 license (BSD-like). Pulled
# in by `libbz2-rs-sys`, the pure-Rust bzip2 implementation.
"bzip2-1.0.6",
# CDLA-Permissive-2.0 is a permissive data license used by `webpki-roots`
# for the Mozilla CA root bundle. Data-only, distribution-compatible.
"CDLA-Permissive-2.0",
@@ -150,12 +145,7 @@ allow = [
confidence-threshold = 0.8
# Per-crate license exceptions: allow a license for a specific crate only,
# rather than globally via the `allow` list above.
exceptions = [
# CDDL-1.0 (copyleft) is pulled in only as a dev/profiling dependency via
# `inferno` -> `pprof` -> `lance-testing`; it is a test dependency that we
# do not distribute, so scope the allowance to `inferno` alone.
{ allow = ["CDDL-1.0"], crate = "inferno" },
]
exceptions = []
# Crates whose license cannot be determined from Cargo metadata but whose
# license we've manually confirmed from upstream. Keep this list minimal.
[[licenses.clarify]]
docs/README.md
+11 -8
View File
@@ -47,22 +47,24 @@ pytest -vv python/tests/docs
### Checking typescript examples
The `@lancedb/lancedb` package must be built before running the tests:
The examples depend on `@lancedb/lancedb` at `file:../dist`, so the package must be
built before running the tests. This uses pnpm; see the
[Typescript contributing guide](../nodejs/CONTRIBUTING.md) for the toolchain setup.
```shell
pushd nodejs
npm ci
npm run build
pnpm install
pnpm build
popd
```
Then you can run the examples by going to the `nodejs/examples` directory and
running the tests like a normal npm package:
Then you can run the examples by going to the `nodejs/examples` directory, which is a
separate pnpm package with its own lockfile:
```shell
pushd nodejs/examples
npm ci
npm test
pnpm install
pnpm test
popd
```
@@ -84,6 +86,7 @@ The new files should be checked into the repository.
```shell
pushd nodejs
npm run docs
# `pnpm docs` would invoke pnpm's built-in `docs` command, not the script.
pnpm run docs
popd
```
docs/openapi.yml
+10 -1
View File
@@ -155,7 +155,7 @@ paths:
vector:
type: FixedSizeList
description: |
The targetted vector to search for. Required.
The targeted vector to search for. Required.
vector_column:
type: string
description: |
@@ -446,6 +446,15 @@ paths:
properties:
column:
type: string
name:
type: string
description: Optional name for the created index.
replace:
type: boolean
default: true
description: |
Whether to replace an existing index with the same resolved
name. Defaults to true.
metric_type:
type: string
nullable: false
docs/package-lock.json
Generated
-135
View File
@@ -1,135 +0,0 @@
{
"name": "lancedb-docs-test",
"version": "1.0.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "lancedb-docs-test",
"version": "1.0.0",
"license": "Apache 2",
"dependencies": {
"apache-arrow": "file:../node/node_modules/apache-arrow",
"vectordb": "file:../node"
},
"devDependencies": {
"@types/node": "^20.11.8",
"typescript": "^5.3.3"
}
},
"../node": {
"name": "vectordb",
"version": "0.21.2-beta.0",
"cpu": [
"x64",
"arm64"
],
"license": "Apache-2.0",
"os": [
"darwin",
"linux",
"win32"
],
"dependencies": {
"@neon-rs/load": "^0.0.74",
"axios": "^1.4.0"
},
"devDependencies": {
"@neon-rs/cli": "^0.0.160",
"@types/chai": "^4.3.4",
"@types/chai-as-promised": "^7.1.5",
"@types/mocha": "^10.0.1",
"@types/node": "^18.16.2",
"@types/sinon": "^10.0.15",
"@types/temp": "^0.9.1",
"@types/uuid": "^9.0.3",
"@typescript-eslint/eslint-plugin": "^5.59.1",
"apache-arrow-old": "npm:apache-arrow@13.0.0",
"cargo-cp-artifact": "^0.1",
"chai": "^4.3.7",
"chai-as-promised": "^7.1.1",
"eslint": "^8.39.0",
"eslint-config-standard-with-typescript": "^34.0.1",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-n": "^15.7.0",
"eslint-plugin-promise": "^6.1.1",
"mocha": "^10.2.0",
"openai": "^4.24.1",
"sinon": "^15.1.0",
"temp": "^0.9.4",
"ts-node": "^10.9.1",
"ts-node-dev": "^2.0.0",
"typedoc": "^0.24.7",
"typedoc-plugin-markdown": "^3.15.3",
"typescript": "^5.1.0",
"uuid": "^9.0.0"
},
"optionalDependencies": {
"@lancedb/vectordb-darwin-arm64": "0.21.2-beta.0",
"@lancedb/vectordb-darwin-x64": "0.21.2-beta.0",
"@lancedb/vectordb-linux-arm64-gnu": "0.21.2-beta.0",
"@lancedb/vectordb-linux-x64-gnu": "0.21.2-beta.0",
"@lancedb/vectordb-win32-x64-msvc": "0.21.2-beta.0"
},
"peerDependencies": {
"@apache-arrow/ts": "^14.0.2",
"apache-arrow": "^14.0.2"
}
},
"../node/node_modules/apache-arrow": {
"version": "14.0.2",
"license": "Apache-2.0",
"dependencies": {
"@types/command-line-args": "5.2.0",
"@types/command-line-usage": "5.0.2",
"@types/node": "20.3.0",
"@types/pad-left": "2.1.1",
"command-line-args": "5.2.1",
"command-line-usage": "7.0.1",
"flatbuffers": "23.5.26",
"json-bignum": "^0.0.3",
"pad-left": "^2.1.0",
"tslib": "^2.5.3"
},
"bin": {
"arrow2csv": "bin/arrow2csv.js"
}
},
"node_modules/@types/node": {
"version": "20.11.8",
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.8.tgz",
"integrity": "sha512-i7omyekpPTNdv4Jb/Rgqg0RU8YqLcNsI12quKSDkRXNfx7Wxdm6HhK1awT3xTgEkgxPn3bvnSpiEAc7a7Lpyow==",
"dev": true,
"dependencies": {
"undici-types": "~5.26.4"
}
},
"node_modules/apache-arrow": {
"resolved": "../node/node_modules/apache-arrow",
"link": true
},
"node_modules/typescript": {
"version": "5.3.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz",
"integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==",
"dev": true,
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
},
"engines": {
"node": ">=14.17"
}
},
"node_modules/undici-types": {
"version": "5.26.5",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
"integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
"dev": true
},
"node_modules/vectordb": {
"resolved": "../node",
"link": true
}
}
}
docs/package.json
-20
View File
@@ -1,20 +0,0 @@
{
"name": "lancedb-docs-test",
"version": "1.0.0",
"description": "auto-generated tests from doc",
"author": "dev@lancedb.com",
"license": "Apache 2",
"dependencies": {
"apache-arrow": "file:../node/node_modules/apache-arrow",
"vectordb": "file:../node"
},
"scripts": {
"build": "tsc -b && cd ../node && npm run build-release",
"example": "npm run build && node",
"test": "npm run build && ls dist/*.js | xargs -n 1 node"
},
"devDependencies": {
"@types/node": "^20.11.8",
"typescript": "^5.3.3"
}
}
docs/src/assets/hero-header.png
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 1.7 MiB

After

Width:  |  Height:  |  Size: 1.3 MiB

docs/src/assets/reverie-banner.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 722 KiB

docs/src/java/java.md
+1 -1
View File
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
<dependency>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-core</artifactId>
<version>0.38.0-beta.11</version>
<version>0.41.0-beta.6</version>
</dependency>
```
docs/src/js/classes/AutoQuery.md
+3 -1
View File
@@ -350,7 +350,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
@@ -417,6 +417,8 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
#### Inherited from
`StandardQueryBase.toArrow`
docs/src/js/classes/BlobFile.md
+158
View File
@@ -0,0 +1,158 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobFile
# Class: BlobFile
A lazy handle to blob bytes. Create one with [Table.fetchBlobFiles](Table.md#fetchblobfiles).
## Methods
### close()
```ts
close(): Promise<void>
```
Releases the handle. Reads after `close()` fail. Calling it again does
nothing.
#### Returns
`Promise`&lt;`void`&gt;
***
### isClosed()
```ts
isClosed(): boolean
```
Returns true after [BlobFile.close](BlobFile.md#close).
#### Returns
`boolean`
***
### read()
```ts
read(maxBytes?, options?): Promise<Buffer>
```
Reads from the cursor and advances the cursor.
Reads to the end when `maxBytes` is omitted, or at most `maxBytes` bytes
otherwise. Returns an empty buffer at the end of the blob.
[BlobFile.readRange](BlobFile.md#readrange) does not move the cursor.
#### Parameters
* **maxBytes?**: `bigint`
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;`Buffer`&gt;
***
### readRange()
```ts
readRange(
start,
end,
options?): Promise<Buffer>
```
Reads the half-open byte range `[start, end)`.
Fails when `end` is past the blob size. Does not move the cursor.
#### Parameters
* **start**: `bigint`
* **end**: `bigint`
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;`Buffer`&gt;
***
### readRanges()
```ts
readRanges(ranges, options?): Promise<Buffer[]>
```
Reads several half-open byte ranges. Returns one buffer per range, in
the order given.
Fails when any `end` is past the blob size. Does not move the cursor.
#### Parameters
* **ranges**: [`BlobRange`](../type-aliases/BlobRange.md)[]
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;`Buffer`[]&gt;
***
### seek()
```ts
seek(position): Promise<void>
```
Moves the cursor to `position`, in bytes from the start of the blob.
#### Parameters
* **position**: `bigint`
#### Returns
`Promise`&lt;`void`&gt;
***
### size()
```ts
size(): bigint
```
Returns the blob size in bytes.
#### Returns
`bigint`
***
### tell()
```ts
tell(): Promise<bigint>
```
Returns the cursor position in bytes.
#### Returns
`Promise`&lt;`bigint`&gt;
docs/src/js/classes/Catalog.md
+118
View File
@@ -0,0 +1,118 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / Catalog
# Class: Catalog
A remote catalog manages databases through the server's root namespace.
## Accessors
### uri
```ts
get uri(): string
```
The root namespace endpoint.
#### Returns
`string`
## Methods
### connectDatabase()
```ts
connectDatabase(name): Promise<Connection>
```
Connect to an existing database by its logical name.
#### Parameters
* **name**: `string`
#### Returns
`Promise`&lt;[`Connection`](Connection.md)&gt;
***
### createDatabase()
```ts
createDatabase(name, options): Promise<Connection>
```
Create a database, or open an existing database when existOk is true.
#### Parameters
* **name**: `string`
* **options** = `{}`
* **options.existOk?**: `boolean`
#### Returns
`Promise`&lt;[`Connection`](Connection.md)&gt;
***
### dropDatabase()
```ts
dropDatabase(name, options): Promise<void>
```
Drop an empty database. The server rejects nonempty databases.
#### Parameters
* **name**: `string`
* **options** = `{}`
* **options.ignoreMissing?**: `boolean`
#### Returns
`Promise`&lt;`void`&gt;
***
### listDatabases()
```ts
listDatabases(options): DatabaseNames
```
Iterate lazily over all database names using `for await...of`.
pageToken resumes from a saved token; omitted starts at the beginning.
pageLimit limits each REST response, not the total; omitted uses the server default.
Request errors are raised during iteration and terminate the iterator.
#### Parameters
* **options** = `{}`
* **options.pageLimit?**: `number`
* **options.pageToken?**: `string`
#### Returns
[`DatabaseNames`](DatabaseNames.md)
#### Example
```ts
for await (const name of catalog.listDatabases({ pageLimit: 20 })) {
console.log(name);
}
```
docs/src/js/classes/Connection.md
+249 -61
View File
@@ -180,13 +180,9 @@ abstract createMaterializedView(
Define a materialized view named `name` over the table `source`.
The view is created empty, with the query recorded in its schema
metadata; `view.refresh()` computes the rows. The view is a normal
table: it can be queried, indexed and searched, and it appears in
`tableNames`. The source table must have stable row ids (create it with
the `newTableEnableStableRowIds` storage option); they keep the view's
provenance valid across source compactions and cannot be enabled after
a table exists. Local databases only.
The view is populated before creation returns. Set `withNoData` to create
only its definition and empty backing table. The view is a normal table:
it can be queried, indexed and searched, and it appears in `tableNames`.
#### Parameters
@@ -202,6 +198,8 @@ a table exists. Local databases only.
* **options.where?**: `string`
* **options.withNoData?**: `boolean`
#### Returns
`Promise`&lt;[`MaterializedView`](MaterializedView.md)&gt;
@@ -317,6 +315,38 @@ Creates a new Table and initialize it with new data.
***
### createView()
```ts
abstract createView(
name,
query,
namespacePath?): Promise<ViewDescription>
```
Create a view: a named query the database plans on every read.
The query is planned once, at creation, so one that cannot be planned is
rejected now rather than at the first read. A view holds no rows, and its
readers see its sources as they are at read time.
There is no replace: a name already taken is an error, and changing a
view is a drop followed by a create.
#### Parameters
* **name**: `string`
* **query**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;[`ViewDescription`](../interfaces/ViewDescription.md)&gt;
***
### describeNamespace()
```ts
@@ -340,6 +370,27 @@ The namespace's properties
***
### describeView()
```ts
abstract describeView(name, namespacePath?): Promise<ViewDescription>
```
What this database records about the view named `name`: its defining
query and the schema that query resolved to.
#### Parameters
* **name**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;[`ViewDescription`](../interfaces/ViewDescription.md)&gt;
***
### display()
```ts
@@ -373,6 +424,54 @@ Drop all tables in the database.
***
### dropMaterializedView()
```ts
abstract dropMaterializedView(name, namespacePath?): Promise<void>
```
Drop the materialized view named `name`.
The view may become unavailable before physical cleanup finishes. Use
[dropMaterializedViewAsync](Connection.md#dropmaterializedviewasync) to retain and wait for the cleanup job.
Rejects a table that exists but is not a materialized view.
#### Parameters
* **name**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;`void`&gt;
***
### dropMaterializedViewAsync()
```ts
abstract dropMaterializedViewAsync(name, namespacePath?): Promise<Job>
```
Start dropping the materialized view named `name` and return its cleanup
job without waiting for completion.
Rejects a table that exists but is not a materialized view.
#### Parameters
* **name**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;[`Job`](Job.md)&gt;
***
### dropNamespace()
```ts
@@ -448,23 +547,50 @@ on the returned job to know when cleanup has finished.
***
### getJob()
### dropView()
```ts
abstract getJob(jobId): Promise<null | JobDescription>
abstract dropView(name, namespacePath?): Promise<void>
```
Describe a single server-side job by id.
Drop the view named `name` and wait for its definition to be deleted.
Resolves to `null` when the server has no such job.
The tables it reads are untouched: a view holds no rows of its own. Use
[dropViewAsync](Connection.md#dropviewasync) to retain the cleanup job instead of waiting on it.
#### Parameters
* **jobId**: `string`
* **name**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;`null` \| [`JobDescription`](../interfaces/JobDescription.md)&gt;
`Promise`&lt;`void`&gt;
***
### dropViewAsync()
```ts
abstract dropViewAsync(name, namespacePath?): Promise<Job>
```
Start dropping the view named `name` and return the job deleting its
definition, without waiting for completion.
The name is free before this resolves. When nothing was bound to it, the
returned job is already finished and has no id.
#### Parameters
* **name**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;[`Job`](Job.md)&gt;
***
@@ -482,75 +608,45 @@ Return true if the connection has not been closed
***
### job()
```ts
abstract job(jobId): Job
```
A [Job](Job.md) handle for a server-side job by id.
The handle is constructed without a server round trip; an unknown id
surfaces when the handle is used. Dropping the handle has no effect on
the job itself.
#### Parameters
* **jobId**: `string`
#### Returns
[`Job`](Job.md)
***
### jobHistory()
```ts
abstract jobHistory(jobId?): Promise<Table<any>>
```
The lifecycle event history of a server-side job, as an Arrow table.
Lists history across all jobs when `jobId` is omitted.
#### Parameters
* **jobId?**: `string`
#### Returns
`Promise`&lt;`Table`&lt;`any`&gt;&gt;
***
### listJobs()
```ts
abstract listJobs(): Promise<JobInfo[]>
abstract listJobs(options?): Listing<JobInfo>
```
List server-side jobs across the database's tables.
Iterate lazily over server-side jobs across the database's tables.
Use `for await...of`. Options control page size and the starting token.
See [Listing](Listing.md) for pagination state and error behavior.
#### Parameters
* **options?**: [`ListingOptions`](../interfaces/ListingOptions.md)
#### Returns
`Promise`&lt;[`JobInfo`](../interfaces/JobInfo.md)[]&gt;
[`Listing`](Listing.md)&lt;[`JobInfo`](../interfaces/JobInfo.md)&gt;
***
### listMaterializedViews()
```ts
abstract listMaterializedViews(): Promise<string[]>
abstract listMaterializedViews(options?): Listing<string>
```
The names of the materialized views in this database.
Found by reading every table's schema, so this costs an open per table.
Iterate lazily with `for await...of`. Local listings inspect table schemas
one page at a time. Options control page size and the starting token.
See [Listing](Listing.md) for pagination state and error behavior.
#### Parameters
* **options?**: [`ListingOptions`](../interfaces/ListingOptions.md)
#### Returns
`Promise`&lt;`string`[]&gt;
[`Listing`](Listing.md)&lt;`string`&gt;
***
@@ -648,6 +744,54 @@ A page of table names and an
***
### listViews()
```ts
abstract listViews(namespacePath?, options?): Listing<string>
```
The names of the views in one namespace.
Names only; a definition comes from [describeView](Connection.md#describeview).
Iterate lazily with `for await...of`. See [Listing](Listing.md) for pagination
state and error behavior; options control page size and the starting token.
#### Parameters
* **namespacePath?**: `string`[]
* **options?**: [`ListingOptions`](../interfaces/ListingOptions.md)
#### Returns
[`Listing`](Listing.md)&lt;`string`&gt;
***
### openJob()
```ts
abstract openJob(jobId): Promise<Job>
```
Open a server-side job by id, returning a handle with its record already
populated. Rejects when the server has no such job, the way
[Connection.openTable](Connection.md#opentable) does for a missing table.
The returned [Job](Job.md) answers for its own state, specification,
result, failure and event history, so there is no separate
connection-level call for any of them.
#### Parameters
* **jobId**: `string`
#### Returns
`Promise`&lt;[`Job`](Job.md)&gt;
***
### openMaterializedView()
```ts
@@ -691,6 +835,28 @@ abstract openTable(
***
### pauseJob()
```ts
abstract pauseJob(jobId): Promise<string>
```
Pause a server-side job by id.
The job's workers drain and it stays parked until resumed. Resolves to
"pausing", "already_paused", or "committing" -- a job finalizing its
results cannot be parked; retry shortly.
#### Parameters
* **jobId**: `string`
#### Returns
`Promise`&lt;`string`&gt;
***
### renameTable()
```ts
@@ -724,6 +890,28 @@ a "not supported" error.
***
### resumeJob()
```ts
abstract resumeJob(jobId): Promise<string>
```
Resume a paused server-side job by id.
Its workers pick their work back up from checkpoints. Resolves to
"resumed", "still_pausing" -- the pause's worker drain is not confirmed
yet; retry shortly -- or "not_paused".
#### Parameters
* **jobId**: `string`
#### Returns
`Promise`&lt;`string`&gt;
***
### ~~tableNames()~~
#### tableNames(options)
docs/src/js/classes/DatabaseNames.md
+79
View File
@@ -0,0 +1,79 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / DatabaseNames
# Class: DatabaseNames
A lazy async iterator of database names with pagination state.
Returned by [Catalog.listDatabases](Catalog.md#listdatabases).
## Implements
- `AsyncIterableIterator`&lt;`string`&gt;
## Methods
### \[asyncIterator\]()
```ts
asyncIterator: AsyncIterableIterator<string>
```
#### Returns
`AsyncIterableIterator`&lt;`string`&gt;
#### Implementation of
`AsyncIterableIterator.[asyncIterator]`
***
### next()
```ts
next(): Promise<IteratorResult<string, any>>
```
Fetch the next name, requesting another page only when needed.
#### Returns
`Promise`&lt;`IteratorResult`&lt;`string`, `any`&gt;&gt;
#### Implementation of
`AsyncIterableIterator.next`
***
### numPageResults()
```ts
numPageResults(): number
```
Number of names available without another REST request.
#### Returns
`number`
***
### pageToken()
```ts
pageToken(): undefined | string
```
Token for the next REST request. Initially this is the supplied starting token.
Returns undefined after the final page is fetched, even if names remain cached.
Drain the cache before saving a token to avoid skipping names when resuming.
A failed request terminates iteration and retains its token for a new iterator.
#### Returns
`undefined` \| `string`
docs/src/js/classes/Job.md
+159 -12
View File
@@ -8,19 +8,46 @@
A handle to an operation that may still be running.
## Constructors
The operation may already be complete when the handle is created.
### new Job()
The detail getters read what the handle last observed. Submitting an
operation returns only a job id, so populating them eagerly would cost an
extra round trip on every call:
- [Job.refresh](Job.md#refresh) and [Job.status](Job.md#status) fetch the whole record.
- [Job.wait](Job.md#wait) records the terminal state it establishes, but not the
rest of the record.
- Everything is null until one of those runs.
## Accessors
### creationMs
```ts
new Job(): Job
get creationMs(): null | number
```
When the job was created, in milliseconds since the epoch.
#### Returns
[`Job`](Job.md)
`null` \| `number`
## Accessors
***
### failure
```ts
get failure(): null | JobFailureInfo
```
Why the job failed, when it failed and the server reports a reason.
#### Returns
`null` \| [`JobFailureInfo`](../interfaces/JobFailureInfo.md)
***
### id
@@ -28,8 +55,69 @@ new Job(): Job
get id(): null | string
```
Identifies the operation on the server that is running it. Operations
that run in this process have no server id. The value is opaque.
Identifies the operation on the server that is running it.
Operations that run in this process have no server id. The value is
opaque: parsing it or storing it to resume the job later is not supported.
#### Returns
`null` \| `string`
***
### jobType
```ts
get jobType(): null | string
```
The job's type, as the server names it. Null for an in-process job, which
has no server-side record.
#### Returns
`null` \| `string`
***
### result
```ts
get result(): any
```
The job-type-specific terminal result. Null until the job succeeds, so a
job that never terminates reports its progress through [Job.events](Job.md#events)
instead.
#### Returns
`any`
***
### spec
```ts
get spec(): any
```
The job-type-specific specification it was submitted with.
#### Returns
`any`
***
### state
```ts
get state(): null | string
```
The last observed lifecycle state, without contacting the backend.
#### Returns
@@ -51,18 +139,61 @@ Request cancellation. Cancelling a finished operation is a no-op.
***
### events()
```ts
events(options?): Promise<Table<any>>
```
This job's recorded lifecycle events.
Where the getters above report a terminal result only once the job reaches
one, events are written as the job runs and outlive the workers that
produced them. A distributed job records a `claim`/`claim_complete` pair
per unit of work, each carrying `rows_processed`, so a job that never
finishes still accounts for what it did.
The server caps results at 1000 rows by default and 10,000 at most, and
truncates without saying so, so pass `limit` for a job that emits an event
per fragment. `filter` is a SQL-like expression over the `state`,
`updated_by`, `emitted_from`, `emitted_by`, and `claim_entity` columns.
#### Parameters
* **options?**: [`JobEventsOptions`](../interfaces/JobEventsOptions.md)
#### Returns
`Promise`&lt;`Table`&lt;`any`&gt;&gt;
***
### refresh()
```ts
refresh(): Promise<void>
```
Ask the backend for this job's current state, and for a server-side job
its full record, then cache it for the getters above.
#### Returns
`Promise`&lt;`void`&gt;
***
### status()
```ts
status(): Promise<string>
```
The operation's current lifecycle state: "running", "finished",
"failed", or "cancelled".
The operation's current lifecycle state: "running", "finished", "failed",
or "cancelled".
A point snapshot; unlike [Job.wait](Job.md#wait) it does not block or reject
on a terminal failure state. States a newer server reports that this
client version does not know pass through as-is.
A point snapshot; unlike [Job.wait](Job.md#wait) it does not block or reject on a
terminal failure state. Also refreshes the getters above.
#### Returns
@@ -70,6 +201,22 @@ client version does not know pass through as-is.
***
### toString()
```ts
toString(): string
```
Every field the handle currently knows, one per line, with the JSON
payloads indented -- a refresh job's spec and result are the point of
printing it.
#### Returns
`string`
***
### wait()
```ts
docs/src/js/classes/Listing.md
+91
View File
@@ -0,0 +1,91 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / Listing
# Class: Listing&lt;T&gt;
A lazy async iterator over database resources with pagination state.
Requests occur during iteration; request errors terminate the iterator.
## Example
```ts
for await (const name of connection.listViews([], { pageLimit: 20 })) {
console.log(name);
}
```
## Type Parameters
• **T**
## Implements
- `AsyncIterableIterator`&lt;`T`&gt;
## Methods
### \[asyncIterator\]()
```ts
asyncIterator: AsyncIterableIterator<T>
```
#### Returns
`AsyncIterableIterator`&lt;`T`&gt;
#### Implementation of
`AsyncIterableIterator.[asyncIterator]`
***
### next()
```ts
next(): Promise<IteratorResult<T, any>>
```
Fetch the next item, requesting another page only when needed.
#### Returns
`Promise`&lt;`IteratorResult`&lt;`T`, `any`&gt;&gt;
#### Implementation of
`AsyncIterableIterator.next`
***
### numPageResults()
```ts
numPageResults(): number
```
Number of cached items available without another request.
#### Returns
`number`
***
### pageToken()
```ts
pageToken(): undefined | string
```
Token for the next request. Initially this is the supplied starting token.
Returns undefined after fetching the final page, even with cached items.
Drain the cache before saving a token to avoid skipping items on resumption.
Failed requests terminate iteration and retain their token for a new iterator.
#### Returns
`undefined` \| `string`
docs/src/js/classes/MaterializedView.md
+4 -11
View File
@@ -49,7 +49,7 @@ get name(): string
definition(): Promise<MaterializedViewDefinition>
```
The query that defines the view, read from its stored schema.
The query that defines the view.
#### Returns
@@ -65,21 +65,14 @@ refresh(options?): Promise<RefreshMaterializedViewResult>
Recompute the view from its source.
The refresh is incremental when the source's changes can be reconciled
into the view -- rows added, changed or removed since the last one --
and otherwise rebuilds. `full` forces a rebuild; `sourceVersion`
refreshes to that source version instead of the latest.
Concurrent refreshes of one view do not duplicate its rows. Two that
plan the same source rows conflict on commit, and the loser throws
rather than writing them a second time.
Every refresh rebuilds the complete view. For local views,
`sourceVersion` refreshes to that source version instead of the latest;
remote SQL refreshes do not support source-version pinning.
#### Parameters
* **options?**
* **options.full?**: `boolean`
* **options.sourceVersion?**: `number`
#### Returns
docs/src/js/classes/MergeInsertBuilder.md
+1 -1
View File
@@ -141,7 +141,7 @@ Currently this causes multiple copies of the row to be created
but that behavior is subject to change.
An optional condition may be specified. If it is, then only
matched rows that satisfy the condtion will be updated. Any
matched rows that satisfy the condition will be updated. Any
rows that do not satisfy the condition will be left as they
are. Failing to satisfy the condition does not cause a
"matched row" to become a "not matched" row.
docs/src/js/classes/OAuthSession.md
+105
View File
@@ -0,0 +1,105 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / OAuthSession
# Class: OAuthSession
Explicit OAuth session lifecycle for the persistent token cache: eager
`login`, non-secret `status`, and local `logout`.
A session is built from the same [OAuthConfig](../interfaces/OAuthConfig.md) used to connect
(including its `tokenCache` options). A connection created with the same
configuration shares the cache, so logging in here prepares tokens for
later processes without any database request.
`login` always runs the configured interactive flow and replaces the cached
session (the most recent login wins). `logout` removes only the local
credential; it does not revoke anything with the provider and does not sign
out of a browser SSO session.
## Example
```typescript
const config: OAuthConfig = {
issuerUrl: "https://issuer.example.com",
clientId: "my-app",
scopes: ["openid", "offline_access"],
flow: OAuthFlowType.DeviceCode,
tokenCache: { cacheDir: "/tmp/my-app/oauth-cache" },
};
const session = new OAuthSession(config);
const status = await session.login();
```
## Constructors
### new OAuthSession()
```ts
new OAuthSession(config): OAuthSession
```
Create a session manager for the given OAuth configuration.
#### Parameters
* **config**: [`OAuthConfig`](../interfaces/OAuthConfig.md)
#### Returns
[`OAuthSession`](OAuthSession.md)
## Methods
### login()
```ts
login(): Promise<SessionStatus>
```
Eagerly run the configured authentication flow and store the session.
A successful login always replaces any prior cached session for this
identity; if the provider does not issue a refresh token (for example
without `offline_access`), the previous record is removed and the status
reports `refreshable == false`.
#### Returns
`Promise`&lt;[`SessionStatus`](../interfaces/SessionStatus.md)&gt;
***
### logout()
```ts
logout(): Promise<SessionLogout>
```
Remove the matching local cached credential.
This only deletes the local cache entry. It does not revoke the refresh
token with the provider and does not sign out of a browser SSO session.
Repeated calls succeed; `removed` reports whether a credential existed.
#### Returns
`Promise`&lt;[`SessionLogout`](../interfaces/SessionLogout.md)&gt;
***
### status()
```ts
status(): Promise<SessionStatus>
```
Report whether a matching cached session exists, with safe metadata.
This never contacts the identity provider and never exposes token values.
#### Returns
`Promise`&lt;[`SessionStatus`](../interfaces/SessionStatus.md)&gt;
docs/src/js/classes/Query.md
+3 -1
View File
@@ -424,7 +424,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
@@ -491,6 +491,8 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
#### Inherited from
`StandardQueryBase.toArrow`
docs/src/js/classes/QueryBase.md
+3 -1
View File
@@ -193,7 +193,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
@@ -252,6 +252,8 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
***
### withRowId()
docs/src/js/classes/Table.md
+134 -16
View File
@@ -74,10 +74,10 @@ now: the column is committed with no values, and rows get them from
[Table#refreshColumn](Table.md#refreshcolumn). Declaring one therefore costs the same on a
large table as on an empty one.
A refresh does not revisit rows it has already filled, so mutating an
input leaves the value computed at fill time; recomputing means dropping
the column and declaring it again. While a declaration reads a column,
that column cannot be renamed, retyped or dropped.
A refresh also recomputes the rows whose inputs changed since they were
computed, so a mutated input is reflected by the next refresh. While a
declaration reads a column, that column cannot be renamed, retyped or
dropped.
On LanceDB Cloud and Enterprise the expression is planned by the
server, and the refresh runs as a server job -- see
@@ -137,6 +137,20 @@ containing the new version number of the table after altering the columns.
***
### blobColumns()
```ts
abstract blobColumns(): Promise<string[]>
```
Blob v2 columns, including nested dotted paths.
#### Returns
`Promise`&lt;`string`[]&gt;
***
### branches()
```ts
@@ -499,6 +513,68 @@ Drop an index from the table.
***
### fetchBlobFiles()
```ts
abstract fetchBlobFiles(
column,
rowIds,
options?): Promise<(null | BlobFile)[]>
```
Opens lazy blob handles for `column` at the given row IDs using the
table's current checkout.
Preserves input order, duplicates, and nulls. Use this for large payloads.
See [Table.fetchBlobs](Table.md#fetchblobs) for row-ID validity across versions.
#### Parameters
* **column**: `string`
* **rowIds**: readonly (`number` \| `bigint`)[]
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;(`null` \| [`BlobFile`](BlobFile.md))[]&gt;
***
### fetchBlobs()
```ts
abstract fetchBlobs(
column,
rowIds,
options?): Promise<(null | Buffer)[]>
```
Bytes for `column` at row IDs from [Query.withRowId](Query.md#withrowid).
Reads the table's current checkout. IDs from another version can fail after
compaction unless stable row ids are enabled. Results keep input order and
duplicates. Null blobs are `null`. Empty blobs are empty buffers.
Remote servers limit each request to 1024 row IDs and 64 MiB of blob bytes.
The client splits requests automatically and reads an individual larger
blob through the Range route. This method still materializes all bytes in
memory; use [Table.fetchBlobFiles](Table.md#fetchblobfiles) for large values.
#### Parameters
* **column**: `string`
* **rowIds**: readonly (`number` \| `bigint`)[]
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;(`null` \| `Buffer`)[]&gt;
***
### flushLsm()
```ts
@@ -516,6 +592,41 @@ so this is safe to call repeatedly.
***
### functionErrors()
```ts
abstract functionErrors(options?): Promise<FunctionErrors>
```
The per-row errors Function refreshes recorded on this table.
A refresh running under a skip policy records each row it skipped with
the input that failed and the error. This lists those records, newest
job first, plus a summary for any fragment whose per-row detail was
capped. LanceDB Cloud and Enterprise only; reading errors needs read
access to the table, since a message carries the value that failed.
#### Parameters
* **options?**: [`FunctionErrorsOptions`](../interfaces/FunctionErrorsOptions.md)
Optional filters: `jobId`,
`column`, and `limit` (server default 10000, cap 100000).
#### Returns
`Promise`&lt;[`FunctionErrors`](../interfaces/FunctionErrors.md)&gt;
The records, the capped fragments,
and whether the listing stopped at its limit.
#### Example
```ts
const { records, truncated } = await table.functionErrors({ column: "embedding" });
```
***
### getLsmStats()
```ts
@@ -551,10 +662,9 @@ Read the [LsmWriteSpec](../interfaces/LsmWriteSpec.md) currently installed on th
Resolves to `undefined` when the MemWAL LSM write path is not enabled (no
spec has been set, or it was removed with [Table#unsetLsmWriteSpec](Table.md#unsetlsmwritespec)).
The returned spec mirrors what was passed to
[Table#setLsmWriteSpec](Table.md#setlsmwritespec), except that `maintainedIndexes` always
reports the concrete list resolved when the spec was set — `undefined`
never round-trips.
The spec is the one installed, including its maintained-index selection:
an absent `maintainedIndexes` for every index the table has, an empty
array for none.
#### Returns
@@ -676,9 +786,17 @@ List all the versions of the table
abstract mergeInsert(on): MergeInsertBuilder
```
Create a [MergeInsertBuilder](MergeInsertBuilder.md), which combines new data with the
existing table in a single transaction — inserting, updating and deleting
rows depending on how they match.
#### Parameters
* **on**: `string` \| `string`[]
The column, or columns, to match source rows against target
rows on. Typically a key or id column. Several columns match on the
composite key: a source row updates a target row only when it agrees on
every one of them.
#### Returns
@@ -846,10 +964,10 @@ abstract refreshColumn(column): Promise<RefreshColumnResult>
Fill the rows of a computed column that hold no value yet.
Rows appended since the last refresh are filled by the next one; rows
already filled are left as they are, so the call is idempotent and does
not observe a mutated input. Local tables only: a remote refresh runs
as a server job, through [Table#refreshColumnAsync](Table.md#refreshcolumnasync).
Rows appended since the last refresh are filled by the next one, and
rows whose inputs changed since they were computed are recomputed;
everything else is left as it is. Local tables only: a remote refresh
runs as a server job, through [Table#refreshColumnAsync](Table.md#refreshcolumnasync).
#### Parameters
@@ -988,8 +1106,8 @@ All variants require the table to have an unenforced primary key
([Table#setUnenforcedPrimaryKey](Table.md#setunenforcedprimarykey)); bucket sharding additionally
requires it to be the single column being bucketed.
Omitting `maintainedIndexes` maintains every index on the table, resolved
here, failing if one cannot be maintained — name them to install anyway.
Omitting `maintainedIndexes` maintains every index the table has,
including ones created later, and skips a kind the MemWAL cannot maintain.
Naming them pins an exact set, and a still-building index is rejected
rather than quietly omitted.
@@ -1258,11 +1376,11 @@ value is 0")
Note: if your condition is something like "some_id_column == 7" and
you are updating many rows (with different ids) then you will get
better performance with a single [`merge_insert`] call instead of
repeatedly calilng this method.
repeatedly calling this method.
##### Parameters
* **updates**: `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **updates**: `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
the
columns to update
docs/src/js/classes/TakeQuery.md
+3 -1
View File
@@ -200,7 +200,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
@@ -267,6 +267,8 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
#### Inherited from
[`QueryBase`](QueryBase.md).[`toArrow`](QueryBase.md#toarrow)
docs/src/js/classes/VectorQuery.md
+42 -7
View File
@@ -447,6 +447,8 @@ nprobes(nprobes): VectorQuery
Set the number of partitions to search (probe)
The number of probes must be greater than 0.
This argument is only used when the vector column has an IVF PQ index.
If there is no index then this value is ignored.
@@ -458,17 +460,16 @@ exhaustiely searched to find matches. This parameter controls how many
partitions should be searched.
Increasing this value will increase the recall of your query but will
also increase the latency of your query. The default value is 20. This
default is good for many cases but the best value to use will depend on
your data and the recall that you need to achieve.
also increase the latency of your query. If this method is not called,
Lance's adaptive probe defaults are used.
For best results we recommend tuning this parameter with a benchmark against
your actual data to find the smallest possible value that will still give
you the desired recall.
For more fine grained control over behavior when you have a very narrow filter
you can use `minimumNprobes` and `maximumNprobes`. This method sets both
the minimum and maximum to the same value.
The value is retained as `nprobes` through client and server request
construction. Lance sets both probe bounds to this value. Explicit minimum
or maximum settings can override their respective bounds.
#### Parameters
@@ -645,6 +646,38 @@ rerank(reranker): VectorQuery
***
### searchEffort()
```ts
searchEffort(searchEffort): VectorQuery
```
Set the initial IVF search effort to a finite value in [0, 1].
The default, null, uses Auto (internally 0.5). Zero starts at the caller
minimum; one searches all partitions within the caller maximum. Later
probing may continue when too few candidates are found.
Any explicit effort, including 0.5, overrides nprobes in either call order.
Explicit minimum and maximum bounds still apply. This is not a recall
guarantee and does not change refinement or HNSW ef. The MemWAL LSM
scanner does not yet support this option.
#### Parameters
* **searchEffort**: `null` \| `number`
#### Returns
[`VectorQuery`](VectorQuery.md)
#### Example
```ts
table.vectorSearch([0.1, 0.2]).searchEffort(0.75).limit(10);
```
***
### select()
```ts
@@ -673,7 +706,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
@@ -740,6 +773,8 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
#### Inherited from
`StandardQueryBase.toArrow`
docs/src/js/enumerations/ClientAuthMethod.md
+48
View File
@@ -0,0 +1,48 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / ClientAuthMethod
# Enumeration: ClientAuthMethod
How the client authenticates to the OAuth token endpoint.
The method applies to every OAuth request that carries client
authentication: client-credentials, authorization-code exchange,
refresh-token, and device-authorization requests. The Azure managed
identity flow ignores this option.
## Enumeration Members
### ClientSecretBasic
```ts
ClientSecretBasic: "client_secret_basic";
```
HTTP Basic authentication. This is the RFC 6749 recommended method and
the normal default for confidential clients, including default Okta
applications. Requires `clientSecret`.
***
### ClientSecretPost
```ts
ClientSecretPost: "client_secret_post";
```
Credentials in the request body, for providers configured to require it.
Requires `clientSecret`.
***
### None
```ts
None: "none";
```
No client authentication, for public clients using PKCE or the device
flow. Cannot be combined with `clientSecret`.
docs/src/js/enumerations/OAuthFlowType.md
+20
View File
@@ -10,6 +10,16 @@ OAuth authentication flow types.
## Enumeration Members
### AuthorizationCode
```ts
AuthorizationCode: "authorization_code";
```
Interactive Authorization Code grant, using PKCE by default.
***
### AzureManagedIdentity
```ts
@@ -27,3 +37,13 @@ ClientCredentials: "client_credentials";
```
Client Credentials grant (service-to-service / M2M).
***
### DeviceCode
```ts
DeviceCode: "device_code";
```
Device Authorization grant for CLI and headless environments.
docs/src/js/functions/blob.md
+55
View File
@@ -0,0 +1,55 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / blob
# Function: blob()
```ts
function blob(name, options): Field
```
Declares a `lance.blob.v2` column.
Query results are descriptors, not payload bytes. Use [Table.fetchBlobs](../classes/Table.md#fetchblobs)
or [Table.fetchBlobFiles](../classes/Table.md#fetchblobfiles) to read bytes.
## Parameters
* **name**: `string`
* **options**: [`BlobOptions`](../type-aliases/BlobOptions.md) = `{}`
## Returns
`Field`
## Example
```ts
import { readFile } from "node:fs/promises";
import { Field, Int64, Schema } from "apache-arrow";
import { blob, connect } from "@lancedb/lancedb";
const db = await connect("./data");
const video = await readFile("clip.mp4");
const table = await db.createTable(
"videos",
[{ id: 1n, video }],
{
schema: new Schema([
new Field("id", new Int64()),
blob("video"),
]),
},
);
const rows = await table.query().select(["id"]).withRowId().toArray();
const rowIds = rows.map((row) => row._rowid as bigint);
const bytes = await table.fetchBlobs("video", rowIds);
const [handle] = await table.fetchBlobFiles("video", rowIds);
const size = handle!.size();
const header = await handle!.readRange(0n, size < 65536n ? size : 65536n);
```
docs/src/js/functions/connectCatalog.md
+34
View File
@@ -0,0 +1,34 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / connectCatalog
# Function: connectCatalog()
```ts
function connectCatalog(endpoint, options): Promise<Catalog>
```
Connect to an HTTP(S) catalog endpoint. Catalog requests omit database-selection
headers; opened database connections inherit authentication and client options.
## Parameters
* **endpoint**: `string`
* **options**: [`CatalogOptions`](../interfaces/CatalogOptions.md) = `{}`
## Returns
`Promise`&lt;[`Catalog`](../classes/Catalog.md)&gt;
## Example
```ts
const catalog = await connectCatalog("https://my-server.example", { apiKey: "secret" });
const db = await catalog.createDatabase("analytics", { existOk: true });
for await (const name of catalog.listDatabases({ pageLimit: 20 })) {
console.log(name);
}
```
docs/src/js/functions/isBlobField.md
+22
View File
@@ -0,0 +1,22 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / isBlobField
# Function: isBlobField()
```ts
function isBlobField(field): boolean
```
Checks for the `lance.blob.v2` extension marker. Does not validate the
field's storage type.
## Parameters
* **field**: `Field`&lt;`any`&gt;
## Returns
`boolean`
docs/src/js/functions/makeJsonField.md
+36
View File
@@ -0,0 +1,36 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / makeJsonField
# Function: makeJsonField()
```ts
function makeJsonField(name, nullable): Field
```
Create an Arrow field backed by LanceDB's JSON extension type.
## Parameters
* **name**: `string`
The field name.
* **nullable**: `boolean` = `true`
Whether the field accepts null values.
## Returns
`Field`
## Example
```ts
import { connect, makeJsonField } from "@lancedb/lancedb";
import { Schema } from "apache-arrow";
const schema = new Schema([makeJsonField("metadata")]);
const db = await connect("/path/to/database");
await db.createTable("items", [{ metadata: '{"source":"api"}' }], { schema });
```
docs/src/js/globals.md
+28 -1
View File
@@ -11,6 +11,7 @@
## Enumerations
- [ClientAuthMethod](enumerations/ClientAuthMethod.md)
- [FullTextQueryType](enumerations/FullTextQueryType.md)
- [OAuthFlowType](enumerations/OAuthFlowType.md)
- [Occur](enumerations/Occur.md)
@@ -19,14 +20,18 @@
## Classes
- [AutoQuery](classes/AutoQuery.md)
- [BlobFile](classes/BlobFile.md)
- [BooleanQuery](classes/BooleanQuery.md)
- [BoostQuery](classes/BoostQuery.md)
- [BranchContents](classes/BranchContents.md)
- [Branches](classes/Branches.md)
- [Catalog](classes/Catalog.md)
- [Connection](classes/Connection.md)
- [DatabaseNames](classes/DatabaseNames.md)
- [HeaderProvider](classes/HeaderProvider.md)
- [Index](classes/Index.md)
- [Job](classes/Job.md)
- [Listing](classes/Listing.md)
- [MakeArrowTableOptions](classes/MakeArrowTableOptions.md)
- [MatchQuery](classes/MatchQuery.md)
- [MaterializedView](classes/MaterializedView.md)
@@ -34,6 +39,7 @@
- [MultiMatchQuery](classes/MultiMatchQuery.md)
- [NativeJsHeaderProvider](classes/NativeJsHeaderProvider.md)
- [OAuthHeaderProvider](classes/OAuthHeaderProvider.md)
- [OAuthSession](classes/OAuthSession.md)
- [PermutationBuilder](classes/PermutationBuilder.md)
- [PhraseQuery](classes/PhraseQuery.md)
- [Query](classes/Query.md)
@@ -60,6 +66,7 @@
- [BranchDiff](interfaces/BranchDiff.md)
- [BranchIndexSummary](interfaces/BranchIndexSummary.md)
- [BranchRowCountSummary](interfaces/BranchRowCountSummary.md)
- [CatalogOptions](interfaces/CatalogOptions.md)
- [CherryPickError](interfaces/CherryPickError.md)
- [CherryPickPreview](interfaces/CherryPickPreview.md)
- [CherryPickResult](interfaces/CherryPickResult.md)
@@ -86,6 +93,10 @@
- [FtsToken](interfaces/FtsToken.md)
- [FullTextQuery](interfaces/FullTextQuery.md)
- [FullTextSearchOptions](interfaces/FullTextSearchOptions.md)
- [FunctionErrorFragment](interfaces/FunctionErrorFragment.md)
- [FunctionErrorRecord](interfaces/FunctionErrorRecord.md)
- [FunctionErrors](interfaces/FunctionErrors.md)
- [FunctionErrorsOptions](interfaces/FunctionErrorsOptions.md)
- [HnswPqOptions](interfaces/HnswPqOptions.md)
- [HnswSqOptions](interfaces/HnswSqOptions.md)
- [IndexConfig](interfaces/IndexConfig.md)
@@ -94,13 +105,14 @@
- [IvfFlatOptions](interfaces/IvfFlatOptions.md)
- [IvfPqOptions](interfaces/IvfPqOptions.md)
- [IvfRqOptions](interfaces/IvfRqOptions.md)
- [JobDescription](interfaces/JobDescription.md)
- [JobEventsOptions](interfaces/JobEventsOptions.md)
- [JobFailureInfo](interfaces/JobFailureInfo.md)
- [JobInfo](interfaces/JobInfo.md)
- [ListNamespacesOptions](interfaces/ListNamespacesOptions.md)
- [ListNamespacesResponse](interfaces/ListNamespacesResponse.md)
- [ListTablesOptions](interfaces/ListTablesOptions.md)
- [ListTablesResponse](interfaces/ListTablesResponse.md)
- [ListingOptions](interfaces/ListingOptions.md)
- [LsmStats](interfaces/LsmStats.md)
- [LsmWriteSpec](interfaces/LsmWriteSpec.md)
- [MaterializedViewDefinition](interfaces/MaterializedViewDefinition.md)
@@ -119,6 +131,8 @@
- [RestNamespaceConfig](interfaces/RestNamespaceConfig.md)
- [RetryConfig](interfaces/RetryConfig.md)
- [ScannableOptions](interfaces/ScannableOptions.md)
- [SessionLogout](interfaces/SessionLogout.md)
- [SessionStatus](interfaces/SessionStatus.md)
- [ShuffleOptions](interfaces/ShuffleOptions.md)
- [SplitCalculatedOptions](interfaces/SplitCalculatedOptions.md)
- [SplitHashOptions](interfaces/SplitHashOptions.md)
@@ -130,12 +144,14 @@
- [TableStatistics](interfaces/TableStatistics.md)
- [TimeoutConfig](interfaces/TimeoutConfig.md)
- [TlsConfig](interfaces/TlsConfig.md)
- [TokenCacheOptions](interfaces/TokenCacheOptions.md)
- [TokenResponse](interfaces/TokenResponse.md)
- [TokenizeOptions](interfaces/TokenizeOptions.md)
- [UpdateFieldMetadataResult](interfaces/UpdateFieldMetadataResult.md)
- [UpdateOptions](interfaces/UpdateOptions.md)
- [UpdateResult](interfaces/UpdateResult.md)
- [Version](interfaces/Version.md)
- [ViewDescription](interfaces/ViewDescription.md)
- [WriteExecutionOptions](interfaces/WriteExecutionOptions.md)
- [WriteProgress](interfaces/WriteProgress.md)
@@ -143,8 +159,15 @@
- [AnalyzePlanDistributedMetrics](type-aliases/AnalyzePlanDistributedMetrics.md)
- [BaseTokenizer](type-aliases/BaseTokenizer.md)
- [BlobData](type-aliases/BlobData.md)
- [BlobInput](type-aliases/BlobInput.md)
- [BlobOptions](type-aliases/BlobOptions.md)
- [BlobRange](type-aliases/BlobRange.md)
- [BlobReadOptions](type-aliases/BlobReadOptions.md)
- [BlobUri](type-aliases/BlobUri.md)
- [Data](type-aliases/Data.md)
- [DataLike](type-aliases/DataLike.md)
- [DataTypeLike](type-aliases/DataTypeLike.md)
- [FieldLike](type-aliases/FieldLike.md)
- [IntoSql](type-aliases/IntoSql.md)
- [IntoVector](type-aliases/IntoVector.md)
@@ -158,10 +181,14 @@
## Functions
- [RecordBatchIterator](functions/RecordBatchIterator.md)
- [blob](functions/blob.md)
- [connect](functions/connect.md)
- [connectCatalog](functions/connectCatalog.md)
- [connectNamespace](functions/connectNamespace.md)
- [instrumentLanceDbMetrics](functions/instrumentLanceDbMetrics.md)
- [isBlobField](functions/isBlobField.md)
- [makeArrowTable](functions/makeArrowTable.md)
- [makeJsonField](functions/makeJsonField.md)
- [packBits](functions/packBits.md)
- [permutationBuilder](functions/permutationBuilder.md)
- [tokenize](functions/tokenize.md)
docs/src/js/interfaces/AddDataOptions.md
+16
View File
@@ -10,6 +10,22 @@ Options for adding data to a table.
## Properties
### allowExternalBlobOutsideBases?
```ts
optional allowExternalBlobOutsideBases: boolean;
```
Whether blob URIs outside registered external bases may be written.
Defaults to `false`. This option is supported only for local/native
tables; remote tables return an error when it is enabled. An enabled write
stores the absolute URI reference without registering a base or copying
the external object into the database. The object must remain accessible
when the blob is read later.
***
### mode
```ts
docs/src/js/interfaces/CatalogOptions.md
+81
View File
@@ -0,0 +1,81 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / CatalogOptions
# Interface: CatalogOptions
Options shared by a catalog and the database connections it returns.
## Extends
- `Omit`&lt;`NativeCatalogOptions`, `"oauthConfig"`&gt;
## Properties
### apiKey?
```ts
optional apiKey: string;
```
#### Inherited from
`Omit.apiKey`
***
### clientConfig?
```ts
optional clientConfig: ClientConfig;
```
#### Inherited from
`Omit.clientConfig`
***
### headerProvider?
```ts
optional headerProvider: HeaderProvider | () => Record<string, string> | Promise<Record<string, string>>;
```
Called for each request to supply authentication headers.
***
### oauthConfig?
```ts
optional oauthConfig: OAuthConfig;
```
***
### readConsistencyInterval?
```ts
optional readConsistencyInterval: number;
```
#### Inherited from
`Omit.readConsistencyInterval`
***
### sqlHostOverride?
```ts
optional sqlHostOverride: string;
```
SQL service endpoint inherited by database connections.
#### Inherited from
`Omit.sqlHostOverride`
docs/src/js/interfaces/CherryPickResult.md
+3 -3
View File
@@ -38,9 +38,9 @@ preview: CherryPickPreview;
```ts
status:
| "failed"
| "unknown"
| "ready"
| "failed"
| "notImplemented"
| "cherryPicked";
| "cherryPicked"
| "unknown";
```
docs/src/js/interfaces/ClientConfig.md
+4
View File
@@ -22,6 +22,10 @@ optional extraHeaders: Record<string, string>;
optional idDelimiter: string;
```
The delimiter joining a namespace path and a name into one object
identifier. `"$"` is the only supported value, and leaving this unset is
how to get it; anything else is rejected when the connection is created.
***
### retryConfig?
docs/src/js/interfaces/DropNamespaceOptions.md
+1 -1
View File
@@ -21,7 +21,7 @@ Refuse to drop if non-empty (restrict) or drop recursively (cascade).
### mode?
```ts
optional mode: "fail" | "skip";
optional mode: "skip" | "fail";
```
Whether to skip if the namespace doesn't exist, or fail.
docs/src/js/interfaces/FtsOptions.md
+2
View File
@@ -37,6 +37,8 @@ The following tokenizers are available:
"raw" - Raw tokenizer. This tokenizer does not split the text into tokens and indexes the entire text as a single token.
"code" - Code tokenizer. This tokenizer keeps snake_case and camelCase identifiers as single tokens.
"icu" - ICU dictionary-based word segmentation.
"icu/split" - ICU segmentation with simple-style delimiter splitting.
docs/src/js/interfaces/FunctionErrorFragment.md
+42
View File
@@ -0,0 +1,42 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / FunctionErrorFragment
# Interface: FunctionErrorFragment
A fragment whose per-row error detail was capped: `rowsSkipped` rows
failed, of which only `rowsRecorded` have a record of their own.
## Properties
### fragmentId
```ts
fragmentId: number;
```
***
### jobId
```ts
jobId: string;
```
***
### rowsRecorded
```ts
rowsRecorded: number;
```
***
### rowsSkipped
```ts
rowsSkipped: number;
```
docs/src/js/interfaces/FunctionErrorRecord.md
+92
View File
@@ -0,0 +1,92 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / FunctionErrorRecord
# Interface: FunctionErrorRecord
One row a Function refresh skipped, as the server recorded it.
## Properties
### column
```ts
column: string;
```
***
### createdAtMillis
```ts
createdAtMillis: number;
```
***
### errorMessage
```ts
errorMessage: string;
```
***
### errorType
```ts
errorType: string;
```
***
### fragmentId
```ts
fragmentId: number;
```
***
### function
```ts
function: string;
```
***
### functionVersion
```ts
functionVersion: string;
```
***
### jobId
```ts
jobId: string;
```
***
### rowOffset?
```ts
optional rowOffset: number;
```
The row's offset within the fragment; absent when the fragment's
detail was capped.
***
### tableVersion
```ts
tableVersion: number;
```
docs/src/js/interfaces/FunctionErrors.md
+39
View File
@@ -0,0 +1,39 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / FunctionErrors
# Interface: FunctionErrors
A table's per-row Function errors.
## Properties
### fragments
```ts
fragments: FunctionErrorFragment[];
```
Fragments whose detail was capped.
***
### records
```ts
records: FunctionErrorRecord[];
```
The recorded rows, newest job first.
***
### truncated
```ts
truncated: boolean;
```
Whether the listing stopped at its limit.
docs/src/js/interfaces/FunctionErrorsOptions.md
+39
View File
@@ -0,0 +1,39 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / FunctionErrorsOptions
# Interface: FunctionErrorsOptions
Which per-row Function errors to list; every filter is optional.
## Properties
### column?
```ts
optional column: string;
```
Only errors on this column.
***
### jobId?
```ts
optional jobId: string;
```
Only errors recorded by this job.
***
### limit?
```ts
optional limit: number;
```
At most this many records (server default 10000, cap 100000).
docs/src/js/interfaces/HnswPqOptions.md
+1 -1
View File
@@ -118,7 +118,7 @@ Number of sub-vectors of PQ.
This value controls how much the vector is compressed during the quantization step.
The more sub vectors there are the less the vector is compressed. The default is
the dimension of the vector divided by 16. If the dimension is not evenly divisible
by 16 we use the dimension divded by 8.
by 16 we use the dimension divided by 8.
The above two cases are highly preferred. Having 8 or 16 values per subvector allows
us to use efficient SIMD instructions.
docs/src/js/interfaces/IndexOptions.md
+1 -1
View File
@@ -16,7 +16,7 @@ optional config: Index;
Advanced index configuration
This option allows you to specify a specfic index to create and also
This option allows you to specify a specific index to create and also
allows you to pass in configuration for training the index.
See the static methods on Index for details on the various index types.
docs/src/js/interfaces/IvfPqOptions.md
+1 -1
View File
@@ -112,7 +112,7 @@ Number of sub-vectors of PQ.
This value controls how much the vector is compressed during the quantization step.
The more sub vectors there are the less the vector is compressed. The default is
the dimension of the vector divided by 16. If the dimension is not evenly divisible
by 16 we use the dimension divded by 8.
by 16 we use the dimension divided by 8.
The above two cases are highly preferred. Having 8 or 16 values per subvector allows
us to use efficient SIMD instructions.
docs/src/js/interfaces/JobDescription.md
-66
View File
@@ -1,66 +0,0 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / JobDescription
# Interface: JobDescription
A described job from `Connection.getJob`.
## Properties
### creationMs
```ts
creationMs: number;
```
When the job was created, in milliseconds since the epoch.
***
### failure?
```ts
optional failure: JobFailureInfo;
```
Why the job failed, when the job is failed and the server reports a
reason.
***
### jobId
```ts
jobId: string;
```
***
### jobType
```ts
jobType: string;
```
***
### specJson?
```ts
optional specJson: string;
```
The job-type-specific specification as a JSON string, when present.
***
### state
```ts
state: string;
```
Lifecycle state: "running", "finished", "failed", or "cancelled".
docs/src/js/interfaces/JobEventsOptions.md
+29
View File
@@ -0,0 +1,29 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / JobEventsOptions
# Interface: JobEventsOptions
Which of a job's events [Job.events](../classes/Job.md#events) returns.
## Properties
### filter?
```ts
optional filter: string;
```
SQL-like filter over the event columns.
***
### limit?
```ts
optional limit: number;
```
Maximum event rows to return, up to the server maximum of 10,000.
docs/src/js/interfaces/JobInfo.md
+1 -1
View File
@@ -26,7 +26,7 @@ When the job was created, in milliseconds since the epoch.
jobId: string;
```
The job id -- what `Connection.getJob` and `Connection.cancelJob`
The job id -- what `Connection.openJob` and `Connection.cancelJob`
accept.
***
docs/src/js/interfaces/ListTablesOptions.md
+1
View File
@@ -15,6 +15,7 @@ optional limit: number;
```
An upper bound on how many tables to return.
Defaults to 100. Zero returns an empty page without a continuation token.
A page may hold fewer than this and still not be the last one, so keep
going while the response carries a page token rather than while pages are
docs/src/js/interfaces/ListingOptions.md
+29
View File
@@ -0,0 +1,29 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / ListingOptions
# Interface: ListingOptions
Pagination controls for lazy resource listings.
## Properties
### pageLimit?
```ts
optional pageLimit: number;
```
Maximum results per request, not a limit on the whole listing.
***
### pageToken?
```ts
optional pageToken: string;
```
Starting continuation token; omitted starts at the beginning.
docs/src/js/interfaces/LsmWriteSpec.md
+2 -3
View File
@@ -34,9 +34,8 @@ Bucket and identity variants: the sharding column.
optional maintainedIndexes: string[];
```
Indexes the MemWAL keeps up to date. Omit to maintain every supported
index, resolved on install — a snapshot, so indexes created later are not
maintained. Pass `[]` for none.
Indexes the MemWAL keeps up to date. Omit for every index the table has,
including ones created later. Pass `[]` for none.
***
docs/src/js/interfaces/MaterializedViewDefinition.md
+7 -44
View File
@@ -6,54 +6,17 @@
# Interface: MaterializedViewDefinition
The query that defines a materialized view.
The query that defines a materialized view, as stored:
`SELECT columns FROM [ns.]table [, function(args) AS alias | , UNNEST(column) AS alias]
[WHERE predicate] [GROUP BY expr, ...] [LIMIT n]`. A Function in `FROM`
position yields one row per element it returns.
## Properties
### filter?
### query
```ts
optional filter: string;
query: string;
```
SQL predicate selecting the source rows the view holds.
***
### inputs
```ts
inputs: string[];
```
Source columns the projections and filter read.
***
### limit?
```ts
optional limit: number;
```
Cap on the number of rows the view holds.
***
### projections
```ts
projections: [string, string][];
```
`[output column, SQL expression]` pairs, in view schema order.
***
### sourceTable
```ts
sourceTable: string;
```
Name of the source table, in the same database as the view.
The defining query, in the canonical spelling the server stores.
docs/src/js/interfaces/NativeOAuthConfig.md
+76 -1
View File
@@ -15,6 +15,39 @@ All token acquisition and refresh is handled in the Rust layer.
## Properties
### audience?
```ts
optional audience: string;
```
Optional provider-specific audience for authorization and token requests.
***
### callbackPort?
```ts
optional callbackPort: number;
```
Port for the authorization_code loopback callback server.
***
### clientAuthMethod?
```ts
optional clientAuthMethod: string;
```
How the client authenticates to the token endpoint: "none",
"client_secret_basic", or "client_secret_post". Defaults to
"client_secret_basic" when a client secret is set, and "none" for
public clients.
***
### clientId
```ts
@@ -41,7 +74,8 @@ Client secret (required for client_credentials).
optional flow: string;
```
Authentication flow: "client_credentials" or "azure_managed_identity"
Authentication flow: "client_credentials", "authorization_code",
"device_code", or "azure_managed_identity"
***
@@ -66,6 +100,16 @@ Client ID for user-assigned managed identity (azure_managed_identity).
***
### redirectUri?
```ts
optional redirectUri: string;
```
Loopback redirect URI for authorization_code.
***
### refreshBufferSecs?
```ts
@@ -78,6 +122,16 @@ the TTL, each request refreshes the token.
***
### resource?
```ts
optional resource: string;
```
Optional resource indicator for authorization and token requests.
***
### scopes
```ts
@@ -86,3 +140,24 @@ scopes: string[];
OAuth scopes to request. For Azure managed identity, exactly one scope
or resource is required. For example: `["api://{app_id}/.default"]`
***
### tokenCache?
```ts
optional tokenCache: TokenCacheOptions;
```
Opt in to the persistent token cache so short-lived processes reuse
one session. Only refresh tokens are persisted.
***
### usePkce?
```ts
optional usePkce: boolean;
```
Whether authorization_code uses S256 PKCE (default: true).
docs/src/js/interfaces/OAuthConfig.md
+106
View File
@@ -26,6 +26,18 @@ const config: OAuthConfig = {
};
```
Providers requiring an explicit target can set `resource` and/or `audience`:
```typescript
const targeted: OAuthConfig = {
issuerUrl: "https://issuer.example.com",
clientId: "app-id",
clientSecret: "secret",
scopes: ["read"],
resource: "https://api.example.com",
audience: "lancedb-api",
};
```
```typescript
const config: OAuthConfig = {
issuerUrl: "https://login.microsoftonline.com/{tenant}/v2.0",
@@ -35,8 +47,57 @@ const config: OAuthConfig = {
};
```
The authorization URL is written to stderr before LanceDB tries to open a
browser, so it can be copied in headless environments.
```typescript
const config: OAuthConfig = {
issuerUrl: "https://login.microsoftonline.com/{tenant}/v2.0",
clientId: "app-id",
scopes: ["openid", "api://lancedb-api/access"],
flow: OAuthFlowType.AuthorizationCode,
};
```
Device Authorization writes the verification URL and user code to stderr
before polling begins.
## Properties
### audience?
```ts
optional audience: string;
```
Provider-specific audience, forwarded to authorization and token endpoints,
including refresh requests. Not supported for Azure managed identity.
***
### callbackPort?
```ts
optional callbackPort: number;
```
Port for the AuthorizationCode loopback callback server (default: 8400).
***
### clientAuthMethod?
```ts
optional clientAuthMethod: ClientAuthMethod;
```
How the client authenticates to the token endpoint (default: auto).
With a `clientSecret` the default is `ClientAuthMethod.ClientSecretBasic`,
which matches the RFC 6749 recommendation and the default configuration
of Okta confidential applications; without a secret the client is public
and no client authentication is sent.
***
### clientId
```ts
@@ -88,6 +149,16 @@ Client ID for user-assigned managed identity (AzureManagedIdentity).
***
### redirectUri?
```ts
optional redirectUri: string;
```
Loopback redirect URI for AuthorizationCode.
***
### refreshBufferSecs?
```ts
@@ -100,6 +171,18 @@ the TTL, each request refreshes the token.
***
### resource?
```ts
optional resource: string;
```
Resource indicator (RFC 8707), forwarded verbatim to authorization and token
endpoints, including refresh requests. Must be an absolute URI without a
fragment. Not supported for Azure managed identity.
***
### scopes
```ts
@@ -109,3 +192,26 @@ scopes: string[];
OAuth scopes to request.
For Azure managed identity, exactly one scope or resource is required.
For example: `["api://{app_id}/.default"]`
***
### tokenCache?
```ts
optional tokenCache: TokenCacheOptions;
```
Opt in to the persistent token cache so short-lived processes reuse one
session. Only refresh tokens are persisted. Only supported by
AuthorizationCode and DeviceCode; Azure managed identity is rejected.
Default: unset (memory only).
***
### usePkce?
```ts
optional usePkce: boolean;
```
Protect AuthorizationCode with S256 PKCE (default: true).
docs/src/js/interfaces/OptimizeOptions.md
+2 -1
View File
@@ -26,7 +26,8 @@ const olderThan = new Date();
olderThan.setDate(olderThan.getDate() - 1));
tbl.optimize({cleanupOlderThan: olderThan});
// Delete all versions except the current version
// Delete versions committed before this point. Versions created by the
// optimize call itself are newer than the cutoff and will be retained.
tbl.optimize({cleanupOlderThan: new Date()});
```
docs/src/js/interfaces/RefreshMaterializedViewResult.md
+1 -1
View File
@@ -14,7 +14,7 @@
mode: string;
```
How the view was brought up to date: "rebuild", "incremental" or "no_op".
How the view was brought up to date: "rebuild".
***
docs/src/js/interfaces/SessionLogout.md
+20
View File
@@ -0,0 +1,20 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / SessionLogout
# Interface: SessionLogout
Result of [OAuthSession.logout](../classes/OAuthSession.md#logout).
## Properties
### removed
```ts
removed: boolean;
```
Whether a cached credential was removed. `false` means no matching
session was cached; logout is idempotent.
docs/src/js/interfaces/SessionStatus.md
+94
View File
@@ -0,0 +1,94 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / SessionStatus
# Interface: SessionStatus
Safe, non-secret view of a cached OAuth session, returned by
[OAuthSession.status](../classes/OAuthSession.md#status) and [OAuthSession.login](../classes/OAuthSession.md#login).
## Properties
### audience?
```ts
optional audience: string;
```
Provider-specific audience used to obtain the cached session, if configured.
***
### clientId
```ts
clientId: string;
```
Client ID of the cached session.
***
### flow
```ts
flow: string;
```
Flow that produced the cached session.
***
### issuerUrl
```ts
issuerUrl: string;
```
Canonical issuer URL of the cached session.
***
### obtainedAt?
```ts
optional obtainedAt: number;
```
When the cached session was obtained, as Unix seconds.
***
### refreshable
```ts
refreshable: boolean;
```
Whether a cached session exists that can obtain tokens without
interactive authentication. Because access tokens are not persisted,
this is `true` exactly when a refresh token is cached; the next
connection refreshes with it rather than opening a browser or device
prompt.
***
### resource?
```ts
optional resource: string;
```
Resource indicator used to obtain the cached session, if configured.
***
### scopes
```ts
scopes: string[];
```
Canonical (sorted, de-duplicated) scope set of the cached session.
docs/src/js/interfaces/TableNamesOptions.md
+1 -1
View File
@@ -19,7 +19,7 @@ instead.
optional limit: number;
```
An optional limit to the number of results to return.
The maximum number of names to return. Omitted returns all names; zero returns none.
***
docs/src/js/interfaces/TableStatistics.md
+17
View File
@@ -18,6 +18,23 @@ Statistics on table fragments
***
### numDeletedRows?
```ts
optional numDeletedRows: number;
```
The number of rows marked as deleted across all fragments of the table
These rows are not included in `numRows`, but still occupy space on disk
until the table is compacted, so a large value here indicates that the
table should be optimized. Fragments in which every row was deleted are
dropped outright, so their rows are not counted here.
Absent (`undefined`) when the backend does not report deletion counts.
***
### numIndices
```ts
docs/src/js/interfaces/TokenCacheOptions.md
+41
View File
@@ -0,0 +1,41 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / TokenCacheOptions
# Interface: TokenCacheOptions
Options for the persistent OAuth token cache.
The cache is opt-in: it is only used when set as `tokenCache` on
[OAuthConfig](OAuthConfig.md). Only refresh tokens are persisted, in a private
directory with owner-only permissions, so short-lived processes can reuse
an authenticated session instead of re-prompting on every start.
Multiple identities (issuer, client, scopes, resource, audience, flow, client authentication)
get separate cache entries. Within one identity the most recent login wins.
## Properties
### cacheDir?
```ts
optional cacheDir: string;
```
Directory that holds cached credentials. Defaults to
`$XDG_CACHE_HOME/lancedb/oauth`, `$HOME/.cache/lancedb/oauth` on Unix,
or `%LOCALAPPDATA%\\lancedb\\oauth` on Windows. The directory is created
with owner-only permissions (`0700`) when missing.
***
### lockTimeoutSecs?
```ts
optional lockTimeoutSecs: number;
```
How long to wait for the cross-process refresh lock before failing, in
seconds (default: 30).
docs/src/js/interfaces/ViewDescription.md
+78
View File
@@ -0,0 +1,78 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / ViewDescription
# Interface: ViewDescription
What a database records about one view.
A view holds no rows: it stores the statement that defines it and the
schema that statement resolved to, so a reader sees its sources as they
are at read time.
## Properties
### defaultDatabase
```ts
defaultDatabase: string;
```
The database that unqualified table names in [query](ViewDescription.md#query) resolve against.
***
### defaultNamespacePath
```ts
defaultNamespacePath: string[];
```
The namespace path those unqualified names resolve against; empty is the
root namespace.
Recorded with the view because it outlives the session that declared it:
a reader resolving the query against its own default namespace could read
a different table than the view was defined over.
***
### name
```ts
name: string;
```
The view's name within its namespace.
***
### namespacePath
```ts
namespacePath: string[];
```
The namespace holding the view; empty is the root namespace.
***
### query
```ts
query: string;
```
The defining query, as the database stores it.
***
### schema
```ts
schema: Schema<any>;
```
The schema the defining query resolved to when the view was created.
docs/src/js/namespaces/embedding/classes/EmbeddingFunction.md
+2 -2
View File
@@ -52,7 +52,7 @@ new EmbeddingFunction<T, M>(): EmbeddingFunction<T, M>
### computeQueryEmbeddings()
```ts
computeQueryEmbeddings(data): Promise<number[] | Uint8Array | Float32Array | Float64Array>
computeQueryEmbeddings(data): Promise<Float32Array | Float64Array | Uint8Array | number[]>
```
Compute the embeddings for a single query
@@ -63,7 +63,7 @@ Compute the embeddings for a single query
#### Returns
`Promise`&lt;`number`[] \| `Uint8Array` \| `Float32Array` \| `Float64Array`&gt;
`Promise`&lt;`Float32Array` \| `Float64Array` \| `Uint8Array` \| `number`[]&gt;
***
docs/src/js/namespaces/embedding/classes/TextEmbeddingFunction.md
+2 -2
View File
@@ -37,7 +37,7 @@ new TextEmbeddingFunction<M>(): TextEmbeddingFunction<M>
### computeQueryEmbeddings()
```ts
computeQueryEmbeddings(data): Promise<number[] | Uint8Array | Float32Array | Float64Array>
computeQueryEmbeddings(data): Promise<Float32Array | Float64Array | Uint8Array | number[]>
```
Compute the embeddings for a single query
@@ -48,7 +48,7 @@ Compute the embeddings for a single query
#### Returns
`Promise`&lt;`number`[] \| `Uint8Array` \| `Float32Array` \| `Float64Array`&gt;
`Promise`&lt;`Float32Array` \| `Float64Array` \| `Uint8Array` \| `number`[]&gt;
#### Overrides
docs/src/js/type-aliases/BaseTokenizer.md
+1
View File
@@ -12,6 +12,7 @@ type BaseTokenizer:
| "whitespace"
| "raw"
| "ngram"
| "code"
| "icu"
| "icu/split"
| `jieba/${string}`
docs/src/js/type-aliases/BlobData.md
+16
View File
@@ -0,0 +1,16 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobData
# Type Alias: BlobData
```ts
type BlobData: Buffer | Uint8Array | ArrayBuffer | Blob;
```
Bytes accepted for a blob value. `ArrayBuffer` is wrapped without copying.
`Blob` and `File` are read with `arrayBuffer()`, which only the async write
paths ([Connection.createTable](../classes/Connection.md#createtable), [Table.add](../classes/Table.md#add),
[Table.mergeInsert](../classes/Table.md#mergeinsert)) can do.
docs/src/js/type-aliases/BlobInput.md
+36
View File
@@ -0,0 +1,36 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobInput
# Type Alias: BlobInput
```ts
type BlobInput:
| BlobData
| BlobUri
| object
| object
| null
| undefined;
```
A value accepted for a `lance.blob.v2` column (see [blob](../functions/blob.md)).
Either inline bytes, a URI pointing at external bytes, or a struct that sets
exactly one of `data` or `uri`. `null` and `undefined` write a null blob.
## Example
```ts
import { pathToFileURL } from "node:url";
import type { BlobInput } from "@lancedb/lancedb";
const rows: { id: bigint; image: BlobInput }[] = [
{ id: 1n, image: await (await fetch(url)).arrayBuffer() },
{ id: 2n, image: pathToFileURL("/data/cat.png") },
{ id: 3n, image: { data: new Blob(["hello"]) } },
];
await table.add(rows);
```
docs/src/js/type-aliases/BlobOptions.md
+48
View File
@@ -0,0 +1,48 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobOptions
# Type Alias: BlobOptions
```ts
type BlobOptions: object;
```
## Type declaration
### dedicatedSizeThreshold?
```ts
optional dedicatedSizeThreshold: number;
```
Max payload bytes stored in a packed sidecar before a dedicated file. Must
be a positive safe integer.
### inlineSizeThreshold?
```ts
optional inlineSizeThreshold: number;
```
Max payload bytes kept inline in the data file. Zero is allowed. Must be a
safe integer.
### nullable?
```ts
optional nullable: boolean;
```
Defaults to true.
### packFileSizeThreshold?
```ts
optional packFileSizeThreshold: number;
```
Max bytes in one packed sidecar before starting another. Must be a positive
safe integer.
docs/src/js/type-aliases/BlobRange.md
+27
View File
@@ -0,0 +1,27 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobRange
# Type Alias: BlobRange
```ts
type BlobRange: object;
```
A half-open byte range `[start, end)` for [BlobFile.readRanges](../classes/BlobFile.md#readranges).
## Type declaration
### end
```ts
end: bigint;
```
### start
```ts
start: bigint;
```
docs/src/js/type-aliases/BlobReadOptions.md
+24
View File
@@ -0,0 +1,24 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobReadOptions
# Type Alias: BlobReadOptions
```ts
type BlobReadOptions: object;
```
Options for blob reads.
## Type declaration
### signal?
```ts
optional signal: AbortSignal;
```
Cancels the read. The call rejects with `signal.reason`, and the native
read stops, including in-flight requests to a remote table.
docs/src/js/type-aliases/BlobUri.md
+13
View File
@@ -0,0 +1,13 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobUri
# Type Alias: BlobUri
```ts
type BlobUri: string | URL;
```
A URI accepted for a blob value. A `URL` is stored as its `href`.
docs/src/js/type-aliases/DataTypeLike.md
+18
View File
@@ -0,0 +1,18 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / DataTypeLike
# Type Alias: DataTypeLike
```ts
type DataTypeLike: DataType | object;
```
A `DataType` from any copy or version of apache-arrow.
Arrow 21 brands its classes with `unique symbol` properties, so a type
object from a second copy of the library no longer satisfies the `DataType`
type of this one even though it is structurally identical. Inputs that only
need to be sanitized accept this looser shape instead.
docs/src/python/python.md
+157 -2
View File
@@ -26,8 +26,90 @@ is also an [asynchronous API client](#connections-asynchronous).
::: lancedb.db.DBConnection
Resource listings (`list_secrets`, `list_views`, `list_jobs`, `list_functions`, and
`list_materialized_views`) return lazy iterators. Use `list(db.list_views())` to
collect synchronous results, or `[name async for name in db.list_views()]` with
an asynchronous connection. `page_limit` controls each request, and `page_token`
resumes a listing. Drain the iterator's cached results before saving its token.
::: lancedb.listing.Listing
::: lancedb.Session
## Catalogs (Synchronous)
Remote catalogs manage databases through a server's root namespace. Opened databases
are ordinary connections. Dropping a database requires it to be empty.
::: lancedb.connect_catalog
::: lancedb.catalog.Catalog
::: lancedb.catalog.DatabaseNames
## Remote SQL
Submit SQL against a remote LanceDB database through the connection.
The connected database and `default_namespace_path=["public"]` are used for
unqualified tables. Fully qualified references can still query other databases
and namespaces available to the same deployment. `execute_query` returns a
reader as soon as its initial result stream is available. `execute_query_async`
returns a query handle immediately; use it to inspect progress, open a reader,
or cancel the query. The SQL client is initialized by the first query and
retained for the lifetime of the remote connection. Query ids are random,
connection-scoped references rather than encoded SQL or durable resume tokens:
```python
import lancedb
db = lancedb.connect(
"db://analytics",
api_key="ldb_...",
host_override="https://api.example.com",
sql_host_override="grpc+tls://sql.example.com:10026",
)
reader = db.execute_query(
"""
SELECT events.id, accounts.name
FROM analytics.public.events AS events
JOIN users.public.accounts AS accounts ON events.user_id = accounts.id
""",
default_namespace_path=["public"],
)
for batch in reader:
print(batch.num_rows)
query = db.execute_query_async("SELECT * FROM events")
print(query.id)
print(query.describe().status)
for batch in query.reader():
print(batch.num_rows)
# Values bind to $1 / $name placeholders and travel as Arrow, so a float32
# stays a float32 and a vector stays a compact fixed-size list:
import numpy as np
reader = db.execute_query(
"SELECT id FROM docs ORDER BY distance(vector, $vector) LIMIT $k",
parameters={"vector": np.random.rand(768).astype(np.float32), "k": 10},
)
# The async connection exposes the same lifecycle without blocking:
# async_db = await lancedb.connect_async(
# "db://analytics",
# api_key="ldb_...",
# host_override="https://api.example.com",
# sql_host_override="grpc+tls://sql.example.com:10026",
# )
# reader = await async_db.execute_query("SELECT * FROM events")
# query = await async_db.execute_query_async("SELECT * FROM events")
# description = await async_db.describe_query(query.id)
# async for batch in await query.reader():
# print(batch.num_rows)
# await query.cancel()
```
## Namespaces (Synchronous)
A namespace-backed connection resolves tables through a
@@ -72,6 +154,10 @@ listing a storage directory.
::: lancedb.functions.UdfDefinition
::: lancedb.secrets.EnvVarSecret
::: lancedb.secrets.SecretInfo
::: lancedb.functions.FunctionRegistrationRequest
::: lancedb.functions.FunctionArtifactRequest
@@ -80,6 +166,8 @@ listing a storage directory.
::: lancedb.functions.PythonAdapterSpec
::: lancedb.functions.FunctionImage
::: lancedb.functions.FunctionVersion
::: lancedb.functions.PythonRuntimeSpec
@@ -94,20 +182,46 @@ listing a storage directory.
::: lancedb.functions.OutputMapping
::: lancedb.functions.AssignmentMapping
::: lancedb.functions.FunctionBinding
::: lancedb.functions.RefreshColumnResult
::: lancedb.FunctionErrors
::: lancedb.FunctionErrorRecord
::: lancedb.FunctionErrorFragment
::: lancedb.job.Job
::: lancedb.job.AsyncJob
::: lancedb.job.JobInfo
::: lancedb.job.JobDescription
::: lancedb.job.JobFailureInfo
::: lancedb.sql.Query
::: lancedb.sql.AsyncQuery
::: lancedb.sql.QueryDescription
::: lancedb.sql.QueryParameters
## Materialized Views (Synchronous)
::: lancedb.materialized_view.MaterializedView
::: lancedb.materialized_view.MaterializedViewDefinition
## Views
::: lancedb.view.ViewDescription
## Expressions
Type-safe expression builder for filters and projections. Use these instead
@@ -223,9 +337,13 @@ tokens = list(
Blob columns store large binary values out of line so they can be read lazily
instead of being materialized with the rest of the row.
::: lancedb.blob
`lancedb.BlobType` is `lance.blob.BlobType` when pylance is installed. Without
pylance, LanceDB uses a matching `lance.blob.v2` extension type so blob columns
still work. Queries return descriptors. Call
[`fetch_blob_files`][lancedb.table.Table.fetch_blob_files] for lazy reads or
[`fetch_blobs`][lancedb.table.Table.fetch_blobs] for eager bytes.
::: lancedb.BlobType
::: lancedb.blob
::: lancedb._blob.BlobFile
options:
@@ -245,6 +363,12 @@ instead of being materialized with the rest of the row.
::: lancedb.exceptions.MissingColumnError
::: lancedb.exceptions.JobNotFoundError
::: lancedb.exceptions.JobFailedError
::: lancedb.exceptions.JobCancelledError
## Integrations
## Pydantic
@@ -275,6 +399,29 @@ instead of being materialized with the rest of the row.
## Reranking
`TypeSafeReranker` supports opt-in request batching:
```python
from lancedb.rerankers import TypeSafeReranker
reranker = TypeSafeReranker(batch_size=40, max_concurrency=8)
```
The default `batch_size=1` keeps the query and document in request state and
sends one request per non-null candidate. With `batch_size=40`, 80 non-null
candidates require two requests. `max_concurrency` still limits simultaneous
requests, and the SDK handles retries.
In batched mode, state contains only `{"query": query}`. Each independent
question contains `{"question": instructions, "document": document}` in its
structured instructions, so it sees only its own document and the shared query.
Custom instructions and criteria are kept verbatim: adapt prompts that explicitly
reference request-state fields such as `state.document` before enabling batching.
Null documents retain a zero score without an API call; empty strings are scored.
API failures, mismatched answer IDs, and invalid probabilities raise errors.
Batching changes the payload and can affect model scores; compare quality and
latency on your workload before opting in.
::: lancedb.rerankers
options:
show_root_heading: false
@@ -282,6 +429,12 @@ instead of being materialized with the rest of the row.
## Connections (Asynchronous)
::: lancedb.connect_catalog_async
::: lancedb.catalog.AsyncCatalog
::: lancedb.catalog.AsyncDatabaseNames
Connections represent a connection to a LanceDb database and
can be used to create, list, or open tables.
@@ -289,6 +442,8 @@ can be used to create, list, or open tables.
::: lancedb.db.AsyncConnection
::: lancedb.listing.AsyncListing
## Namespaces (Asynchronous)
::: lancedb.connect_namespace_async
docs/tsconfig.json
-17
View File
@@ -1,17 +0,0 @@
{
"include": [
"src/*.ts",
],
"compilerOptions": {
"target": "es2022",
"module": "nodenext",
"declaration": true,
"outDir": "./dist",
"strict": true,
"allowJs": true,
"resolveJsonModule": true,
},
"exclude": [
"./dist/*",
]
}
docs/web-tests/camelot.json
+66
View File
@@ -0,0 +1,66 @@
[
{
"id": 1,
"name": "King Arthur",
"role": "King of Camelot",
"description": "The legendary ruler of Camelot, wielder of Excalibur, and leader of the Knights of the Round Table.",
"vector": [0.72, -0.28, 0.60, 0.86],
"stats": { "strength": 2, "courage": 5, "magic": 1, "wisdom": 4 }
},
{
"id": 2,
"name": "Merlin",
"role": "Wizard and Advisor",
"description": "A powerful wizard and prophet who mentors Arthur and shapes the destiny of Camelot through magic and foresight.",
"vector": [0.05, 0.88, 0.62, 0.85],
"stats": { "strength": 2, "courage": 4, "magic": 5, "wisdom": 5 }
},
{
"id": 3,
"name": "Queen Guinevere",
"role": "Queen of Camelot",
"description": "Arthur's queen, admired for her grace and diplomacy, whose romances and loyalties influence Camelot's fate.",
"vector": [0.22, -0.22, 0.42, 0.82],
"stats": { "strength": 1, "courage": 3, "magic": 1, "wisdom": 4 }
},
{
"id": 4,
"name": "Sir Lancelot",
"role": "Knight of the Round Table",
"description": "Arthur's most skilled knight, famed for unmatched combat prowess and his tragic love for Queen Guinevere.",
"vector": [0.86, -0.35, 0.38, 0.55],
"stats": { "strength": 5, "courage": 5, "magic": 1, "wisdom": 3 }
},
{
"id": 5,
"name": "Sir Gawain",
"role": "Knight of the Round Table",
"description": "A noble and honorable knight known for his courtesy and his encounter with the Green Knight.",
"vector": [0.82, -0.32, 0.52, 0.60],
"stats": { "strength": 4, "courage": 5, "magic": 1, "wisdom": 4 }
},
{
"id": 6,
"name": "Sir Galahad",
"role": "Knight of the Round Table",
"description": "The purest and most virtuous knight, chosen to achieve the Holy Grail due to his unwavering spiritual purity.",
"vector": [0.80, -0.20, 0.70, 0.78],
"stats": { "strength": 4, "courage": 5, "magic": 2, "wisdom": 5 }
},
{
"id": 7,
"name": "Sir Percival",
"role": "Knight of the Round Table",
"description": "A loyal and innocent knight whose bravery and sincerity make him one of the key seekers of the Holy Grail.",
"vector": [0.78, -0.36, 0.48, 0.52],
"stats": { "strength": 4, "courage": 4, "magic": 1, "wisdom": 3 }
},
{
"id": 8,
"name": "Mordred",
"role": "Traitor Knight",
"description": "Arthur's treacherous son or nephew who ultimately rebels against him, leading to Camelot's downfall.",
"vector": [0.68, -0.30, -0.65, 0.20],
"stats": { "strength": 4, "courage": 2, "magic": 1, "wisdom": 2 }
}
]
docs/web-tests/mdx_snippets_gen.py
+362
View File
@@ -0,0 +1,362 @@
"""
Generate MDX-ready snippet modules that export raw string constants.
The generator groups snippets by the originating test file name rather than by
language. For a source file like `tests/py/test_basic_usage.py`, the snippets are
written under `docs/snippets/basic.mdx`. Each exported constant is prefixed with
a language identifier (Py|Ts|Rs) followed by the snippet name converted to
TitleCase so that docs authors can selectively assemble `<CodeGroup>` blocks.
"""
from __future__ import annotations
import argparse
import json
import os
import re
from dataclasses import dataclass
from pathlib import Path
from typing import Dict, Iterable, Iterator, List, Mapping, MutableMapping, Tuple
# Supported languages and relevant metadata.
LANG_ORDER = ("py", "ts", "rs")
LANG_PREFIX = {"py": "Py", "ts": "Ts", "rs": "Rs"}
LANG_EXTENSIONS = {
".py": "py",
".ts": "ts",
".tsx": "ts",
".rs": "rs",
}
LANG_MARKERS = {"py": ["#"], "ts": ["//"], "rs": ["//"]}
# Match each export block; we rely on the blank line we emit after each export to
# avoid accidentally truncating on semicolons inside the JSON string literal.
EXPORT_RE = re.compile(r"export const (\w+)\s*=\s*(.+?);\s*\n(?:\s*\n|$)", re.DOTALL)
DEFAULT_SOURCE_DIRS = (
Path("tests") / "py",
Path("tests") / "ts",
Path("tests") / "rs",
)
@dataclass(frozen=True)
class SnippetRecord:
lang: str
snippet_name: str
export_name: str
text: str
source_rel: str
def build_markers(
comment_markers: Iterable[str],
) -> Tuple[re.Pattern[str], re.Pattern[str]]:
escaped = "|".join(re.escape(c) for c in comment_markers)
start_re = re.compile(rf"^\s*(?:{escaped})\s*--8<--\s*\[start:([^\]]+)\]")
end_re = re.compile(rf"^\s*(?:{escaped})\s*--8<--\s*\[end:([^\]]+)\]")
return start_re, end_re
def parse_snippets(
lines: List[str], start_re: re.Pattern[str], end_re: re.Pattern[str]
) -> Dict[str, List[str]]:
snippets: Dict[str, List[str]] = {}
stack: List[Tuple[str, int]] = []
for idx, line in enumerate(lines):
m_start = start_re.match(line)
if m_start:
name = m_start.group(1).strip()
stack.append((name, idx))
continue
m_end = end_re.match(line)
if m_end:
name = m_end.group(1).strip()
if not stack:
raise ValueError(
f"End marker for '{name}' found at line {idx + 1} without matching start"
)
start_name, start_idx = stack.pop()
if start_name != name:
raise ValueError(
f"Mismatched markers: start '{start_name}' at line {start_idx + 1}, "
f"end '{name}' at line {idx + 1}"
)
snippets[name] = lines[start_idx + 1 : idx]
continue
if stack:
open_names = ", ".join(f"{n}@{i + 1}" for n, i in stack)
raise ValueError(f"Unclosed snippet markers for: {open_names}")
return snippets
def dedent(lines: List[str]) -> List[str]:
min_indent = None
for ln in lines:
if ln.strip() == "":
continue
expanded = ln.expandtabs(4)
indent = len(expanded) - len(expanded.lstrip(" "))
if min_indent is None or indent < min_indent:
min_indent = indent
if not lines:
return []
if min_indent in (None, 0):
return [ln.rstrip("\n") for ln in lines]
dedented: List[str] = []
for ln in lines:
expanded = ln.expandtabs(4)
if expanded.strip() == "":
dedented.append(expanded.rstrip("\n"))
else:
dedented.append(expanded[min_indent:].rstrip("\n"))
return dedented
def ensure_dir(path: Path) -> None:
path.mkdir(parents=True, exist_ok=True)
def detect_language(file_path: Path) -> str:
lang = LANG_EXTENSIONS.get(file_path.suffix.lower())
if not lang:
raise ValueError(f"Unsupported file extension for {file_path}")
return lang
def markers_for_lang(lang: str) -> Tuple[re.Pattern[str], re.Pattern[str]]:
markers = LANG_MARKERS.get(lang)
if not markers:
raise ValueError(f"Unsupported language for markers: {lang}")
return build_markers(markers)
def normalize_target_name(file_path: Path) -> str:
stem = file_path.stem
if stem.startswith("test_"):
stem = stem[len("test_") :]
if stem.endswith("_test"):
stem = stem[: -len("_test")]
if stem.endswith(".test"):
stem = stem[: -len(".test")]
if stem.endswith(".spec"):
stem = stem[: -len(".spec")]
if stem == "":
stem = file_path.stem
return stem.replace(" ", "_")
def to_title_case(name: str) -> str:
parts = re.split(r"[^0-9a-zA-Z]+", name)
filtered = [p for p in parts if p]
return "".join(p[:1].upper() + p[1:].lower() for p in filtered) or "Snippet"
def format_export_name(lang: str, snippet_name: str) -> str:
prefix = LANG_PREFIX[lang]
return f"{prefix}{to_title_case(snippet_name)}"
def snippet_text(lines: List[str]) -> str:
if not lines:
return ""
joined = "\n".join(lines).rstrip("\n")
return (joined + "\n") if joined else ""
def to_js_literal(text: str) -> str:
return json.dumps(text, ensure_ascii=False)
def split_export_prefix(export_name: str) -> Tuple[str | None, str]:
for lang, prefix in LANG_PREFIX.items():
if export_name.startswith(prefix):
suffix = export_name[len(prefix) :]
return lang, suffix or export_name
return None, export_name
def parse_existing_module(path: Path) -> Mapping[str, Mapping[str, SnippetRecord]]:
try:
text = path.read_text(encoding="utf-8")
except FileNotFoundError:
return {}
existing: Dict[str, Dict[str, SnippetRecord]] = {}
for match in EXPORT_RE.finditer(text):
export_name, literal = match.group(1), match.group(2).strip()
lang, snippet_suffix = split_export_prefix(export_name)
if not lang:
continue
try:
decoded = json.loads(literal)
except Exception:
decoded = literal.strip().strip(";")
record = SnippetRecord(
lang=lang,
snippet_name=snippet_suffix or export_name,
export_name=export_name,
text=decoded,
source_rel="(existing module)",
)
existing.setdefault(lang, {})[record.snippet_name] = record
return existing
def iter_source_files(source_dirs: Iterable[Path]) -> Iterator[Path]:
for root in source_dirs:
for path in root.rglob("*"):
if path.is_file() and path.suffix.lower() in LANG_EXTENSIONS:
yield path
def write_if_changed(path: Path, content: str) -> bool:
if path.exists():
try:
current = path.read_text(encoding="utf-8")
if current == content:
return False
except Exception:
pass
ensure_dir(path.parent)
path.write_text(content, encoding="utf-8")
return True
def collect_snippets(
source_dirs: Iterable[Path],
) -> Mapping[str, Mapping[str, Mapping[str, SnippetRecord]]]:
result: Dict[str, Dict[str, Dict[str, SnippetRecord]]] = {}
for file_path in iter_source_files(source_dirs):
lang = detect_language(file_path)
start_re, end_re = markers_for_lang(lang)
try:
text = file_path.read_text(encoding="utf-8")
except Exception as exc: # pragma: no cover
raise RuntimeError(f"Failed to read {file_path}: {exc}") from exc
lines = text.splitlines(keepends=True)
snippets = parse_snippets(lines, start_re, end_re)
if not snippets:
continue
target = normalize_target_name(file_path)
lang_map = result.setdefault(target, {}).setdefault(lang, {})
for snippet_name, content_lines in snippets.items():
if snippet_name in lang_map:
raise ValueError(
f"Duplicate snippet '{snippet_name}' in {file_path} and {lang_map[snippet_name].source_rel}"
)
dedented = dedent(content_lines)
export_name = format_export_name(lang, snippet_name)
source_rel = os.path.relpath(file_path, Path.cwd())
lang_map[snippet_name] = SnippetRecord(
lang=lang,
snippet_name=snippet_name,
export_name=export_name,
text=snippet_text(dedented),
source_rel=source_rel,
)
return result
def render_module(
target: str, lang_map: Mapping[str, Mapping[str, SnippetRecord]]
) -> str:
parts: List[str] = []
parts.append(
"{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */}\n\n"
)
for lang in LANG_ORDER:
snippets = lang_map.get(lang)
if not snippets:
continue
for snippet_name in sorted(snippets.keys()):
record = snippets[snippet_name]
literal = to_js_literal(record.text)
parts.append(f"export const {record.export_name} = {literal};\n\n")
return "".join(parts)
def generate_modules(
snippets_by_target: Mapping[str, Mapping[str, Mapping[str, SnippetRecord]]],
output_root: Path,
) -> None:
modules_written = 0
total_exports = 0
for target in sorted(snippets_by_target.keys()):
module_path = output_root / f"{target}.mdx"
lang_map = snippets_by_target[target]
existing_langs = parse_existing_module(module_path)
for lang, snippets in existing_langs.items():
if lang in lang_map:
continue
print(module_path, " -- ", lang)
merged_lang = lang_map.setdefault(lang, {})
existing_export_names = {rec.export_name for rec in merged_lang.values()}
for snippet_name, record in snippets.items():
# If an export with the same name already exists (e.g., regenerated),
# prefer the newly generated snippet and skip the old one to avoid duplicates.
if record.export_name in existing_export_names:
continue
merged_lang.setdefault(snippet_name, record)
total_exports += sum(len(snippets) for snippets in lang_map.values())
module_content = render_module(target, lang_map)
if write_if_changed(module_path, module_content):
modules_written += 1
print(
f"\nGenerated {total_exports} snippets\n---"
)
def resolve_source_dirs(args_dirs: List[str] | None) -> List[Path]:
if args_dirs:
dirs = [Path(p) for p in args_dirs]
else:
dirs = list(DEFAULT_SOURCE_DIRS)
missing = [str(p) for p in dirs if not p.exists()]
if missing:
raise FileNotFoundError(f"Source directories not found: {', '.join(missing)}")
return dirs
def main() -> int:
parser = argparse.ArgumentParser(
description="Generate MDX snippet modules grouped by test filename."
)
parser.add_argument(
"-s",
"--source-dir",
action="append",
help="Path(s) to language test directories (default: tests/py, ts, rs)",
)
parser.add_argument(
"-o",
"--output-dir",
default=str(Path("docs") / "snippets"),
help="Directory where snippet modules are written (default: docs/snippets)",
)
args = parser.parse_args()
source_dirs = resolve_source_dirs(args.source_dir)
snippets = collect_snippets(source_dirs)
if not snippets:
print("No snippets found.")
return 0
output_root = Path(args.output_dir)
generate_modules(snippets, output_root)
return 0
if __name__ == "__main__":
raise SystemExit(main())
docs/web-tests/py/conftest.py
+56
View File
@@ -0,0 +1,56 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import shutil
from pathlib import Path
import lancedb
import pytest
import pytest_asyncio
from lancedb.db import AsyncConnection, DBConnection
class DatabasePathFactory:
"""Create per-test database directories and ensure they get removed."""
def __init__(self, base_dir: Path) -> None:
self._base_dir = base_dir
self._created: list[Path] = []
def __call__(self, name: str) -> Path:
safe_name = name.replace("/", "_")
path = self._base_dir / safe_name
if path.exists():
shutil.rmtree(path, ignore_errors=True)
path.mkdir(parents=True, exist_ok=True)
self._created.append(path)
return path
def cleanup(self) -> None:
for path in reversed(self._created):
shutil.rmtree(path, ignore_errors=True)
self._created.clear()
@pytest.fixture
def db_path_factory(tmp_path: Path):
factory = DatabasePathFactory(tmp_path)
yield factory
factory.cleanup()
@pytest.fixture
def mem_db() -> DBConnection:
return lancedb.connect("memory://")
@pytest.fixture
def tmp_db(db_path_factory) -> DBConnection:
"""Create a temporary database connection for testing."""
db_path = db_path_factory("tmp_db")
return lancedb.connect(str(db_path))
@pytest_asyncio.fixture
async def mem_db_async() -> AsyncConnection:
return await lancedb.connect_async("memory://")

Some files were not shown because too many files have changed in this diff Show More