rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-12 23:02:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: merge main into fork worker repair

Browse Source 
Retain both test imports and align remote data-loader fixtures with aliased row-offset projections. Verify returned batch values under spawn and fork.
This commit is contained in:
Gatefixer
2026-10-03 04:23:16 +00:00
parent 1825f6df9e 3d8c9e2ddc
commit 132964737a
420 changed files with 96665 additions and 18956 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.agents/skills/README.md
-4
View File
@@ -5,7 +5,3 @@ This directory contains repo-scoped code agent skills for the LanceDB project.
Each skill is a folder that contains a required `SKILL.md` and optional bundled resources.
Codex discovers skills from `.agents/skills` in the current working directory and parent directories.
The `lancedb` skill lives in the `plugins/lancedb` plugin (see `plugins/lancedb/skills/lancedb`)
so it can be installed via the plugin marketplaces (`.claude-plugin/marketplace.json` and
`.agents/plugins/marketplace.json`); the `lancedb` entry here is a symlink into that plugin.
.agents/skills/lancedb
-1
View File
@@ -1 +0,0 @@
../../plugins/lancedb/skills/lancedb
.bumpversion.toml
+1 -1
View File
@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.37.1-beta.0"
current_version = "0.40.0-beta.12"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.
.cargo/config.toml
+12
View File
@@ -9,6 +9,18 @@ debug = true
codegen-units = 16
lto = "thin"
[profile.release-no-lto]
inherits = "release"
debug = true
lto = false
# Prioritize compile time when LTO is not relevant to the measurement.
codegen-units = 16
[profile.bench]
inherits = "release"
lto = "thin"
codegen-units = 16
[target.'cfg(all())']
rustflags = [
"-Wclippy::all",
.github/dependabot.yml
+36
View File
@@ -17,6 +17,18 @@ updates:
# newer minimum versions.
versioning-strategy: lockfile-only
groups:
# The arrow-rs and datafusion crates are released in lockstep and have to
# move together, so keep them in one PR instead of one per sub-crate.
# Listed first: a dependency joins the first group it matches.
arrow-datafusion:
patterns:
- arrow
- arrow-*
- parquet
- parquet-*
- datafusion
- datafusion-*
- object_store
rust-minor-patch:
update-types:
- minor
@@ -32,3 +44,27 @@ updates:
python-deps:
patterns:
- "*"
# The npm ecosystem covers pnpm lockfiles. There are two separate installs:
# the bindings themselves and the examples, which have their own lockfile.
# As with cargo and pip above, only bump the lockfile — the version ranges
# in package.json are our consumers' constraints, not ours.
- package-ecosystem: npm
directory: /nodejs
schedule:
interval: weekly
versioning-strategy: lockfile-only
groups:
nodejs-deps:
patterns:
- "*"
- package-ecosystem: npm
directory: /nodejs/examples
schedule:
interval: weekly
versioning-strategy: lockfile-only
groups:
nodejs-examples-deps:
patterns:
- "*"
.github/workflows/ci-scripts.yml
+30
View File
@@ -0,0 +1,30 @@
name: CI scripts
on:
push:
branches:
- main
paths:
- ci/set_lance_version.py
- ci/tests/**
- .github/workflows/ci-scripts.yml
pull_request:
paths:
- ci/set_lance_version.py
- ci/tests/**
- .github/workflows/ci-scripts.yml
permissions:
contents: read
jobs:
test:
name: Test CI scripts
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: actions/setup-python@v6
with:
python-version: "3.13"
- name: Run tests
run: python -m unittest discover -s ci/tests -v
.github/workflows/codex-update-lance-dependency.yml
+2 -2
View File
@@ -4,14 +4,14 @@ on:
workflow_call:
inputs:
tag:
description: "Tag name from Lance. If omitted, the skill will use the latest Lance release that needs an update."
description: "Tag name from Lance (e.g. `v7.2.0-beta.1`). If omitted, the newest release is resolved automatically — stable releases are preferred over pre-releases — and the run is skipped if it is not newer than the version currently pinned in Cargo.toml."
required: false
default: ""
type: string
workflow_dispatch:
inputs:
tag:
description: "Tag name from Lance. Leave empty to use the latest Lance release that needs an update."
description: "Tag name from Lance (e.g. `v7.2.0-beta.1`). Leave empty to resolve the newest release automatically — stable releases are preferred over pre-releases — and skip the run if it is not newer than the version currently pinned in Cargo.toml."
required: false
default: ""
type: string
.github/workflows/dev.yml
+10 -4
View File
@@ -29,12 +29,14 @@ jobs:
steps:
- uses: actions/setup-node@v6
with:
node-version: "18"
node-version: "24"
- uses: pnpm/action-setup@v6
with:
version: 11.1.1
# These rules are disabled because Github will always ensure there
# is a blank line between the title and the body and Github will
# word wrap the description field to ensure a reasonable max line
# length.
- run: npm install @commitlint/config-conventional
- run: >
echo 'module.exports = {
"rules": {
@@ -43,7 +45,11 @@ jobs:
"body-leading-blank": [0, "always"]
}
}' > .commitlintrc.js
- run: npx commitlint --extends @commitlint/config-conventional --verbose <<< $COMMIT_MSG
- run: >
pnpm dlx
--package @commitlint/cli@21.2.2
--package @commitlint/config-conventional@21.2.2
commitlint --extends @commitlint/config-conventional --verbose <<< $COMMIT_MSG
env:
COMMIT_MSG: >
${{ github.event.pull_request.title }}
@@ -54,7 +60,7 @@ jobs:
with:
script: |
const message = `**ACTION NEEDED**
Lance follows the [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0/) for release automation.
The PR title and description are used as the merge commit message.\
.github/workflows/docs-link-check.yml
+243
View File
@@ -0,0 +1,243 @@
name: Check doc links
# Checking external links is inherently noisy: third-party sites rate-limit
# automated clients, reject non-browser user agents, and go down temporarily.
# Blocking pull requests on that trades a lot of false failures for very little
# signal, so this runs on a schedule and reports findings in a single tracking
# issue instead of failing anyone's build.
on:
schedule:
- cron: "0 7 * * *"
workflow_dispatch:
# The report lives in one repository-global issue, so runs must not overlap: a
# lookup racing a create produces duplicate issues, and a healthy run closing
# the issue while a failing run only rewrites its body would leave a broken
# report closed. The group is deliberately ref-independent so that a manual
# dispatch serializes against the scheduled run.
concurrency:
group: docs-link-check
cancel-in-progress: false
permissions: {}
env:
REPORT_TITLE: "Docs link checker report"
jobs:
scan:
name: Scan links
runs-on: ubuntu-24.04
# lychee-action is pinned by SHA, but its wrapper downloads the lychee
# release tarball at run time without verifying a digest, and hands the
# resulting binary a GitHub token. Release assets remain replaceable, so
# that binary is confined to a job whose token can only read public
# content; everything that writes runs in the report job below.
permissions:
contents: read
outputs:
checker_outcome: ${{ steps.lychee.outcome }}
exit_code: ${{ steps.lychee.outputs.exit_code }}
status: ${{ steps.validate.outputs.status }}
steps:
- name: Checkout
uses: actions/checkout@v6
with:
# workflow_dispatch can run from any ref, but the report is
# repository-global. Always measure the default branch so a manual
# run from a topic branch cannot close a report that main warrants,
# or overwrite it with branch-only findings.
ref: ${{ github.event.repository.default_branch }}
persist-credentials: false
- name: Check links
id: lychee
continue-on-error: true
uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2.9.0
with:
# Restricted to http(s) on purpose. Much of docs/src is generated
# API reference (the js/ tree comes from `pnpm run docs` in nodejs)
# and the hand-written pages use mkdocstrings cross-references and
# nav-relative paths that only resolve in the site mkdocs builds,
# not in this checkout, so relative links would be reported as
# broken on every run.
args: >-
--scheme https
--scheme http
--no-progress
--max-retries 3
--timeout 20
'docs/src/**/*.md'
format: json
output: ./lychee/out.json
jobSummary: false
# The report issue, not a red workflow run, is the signal for link
# findings and checker failures alike.
fail: false
- name: Validate report
id: validate
# lychee does not reserve exit code 2 for broken links: its CLI
# parser also exits 2 on an invalid option, before any link was
# checked or any report written. Only a parseable report whose
# counts agree with a completed exit code (0 or 2) counts as a link
# verdict. Everything else becomes a checker-error report instead of
# failing the workflow. Exit 2 covers timeouts as well as errors, and a
# timed-out host is exactly the transient unavailability this report
# exists to surface, so both count as findings. Requiring total > 0
# also catches a glob that silently stopped matching any file.
if: always()
env:
CHECKER_OUTCOME: ${{ steps.lychee.outcome }}
EXIT_CODE: ${{ steps.lychee.outputs.exit_code }}
run: |
status=checker-error
if [[ "$CHECKER_OUTCOME" == success ]] &&
[[ "$EXIT_CODE" == 0 || "$EXIT_CODE" == 2 ]] &&
jq -e --argjson code "$EXIT_CODE" '
(.total > 0) and
(if $code == 0
then .errors == 0 and .timeouts == 0
and (.error_map | length == 0) and (.timeout_map | length == 0)
else (.errors + .timeouts) > 0
and ((.error_map | length) + (.timeout_map | length)) > 0
end)
' ./lychee/out.json
then
if [[ "$EXIT_CODE" == 0 ]]; then
status=healthy
else
status=findings
fi
fi
echo "status=$status" >> "$GITHUB_OUTPUT"
echo "Validated link check as $status"
- name: Upload report
if: steps.validate.outputs.status == 'findings'
uses: actions/upload-artifact@v7
with:
name: link-report
path: ./lychee/out.json
retention-days: 7
report:
name: Update report issue
needs: scan
runs-on: ubuntu-24.04
# Deliberately no checkout: this job needs the report artifact and the
# issues API, not the repository contents.
permissions:
issues: write
env:
CHECKER_OUTCOME: ${{ needs.scan.outputs.checker_outcome }}
EXIT_CODE: ${{ needs.scan.outputs.exit_code }}
STATUS: ${{ needs.scan.outputs.status }}
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
steps:
- name: Find existing report issue
id: report
# Matched on title alone, and through search rather than a listing:
# the issue action applies labels in a separate call after creating the
# issue, so a label filter misses a half-created report, and this
# repository has far more open issues than one listing page holds.
# Closed issues are included because a healthy run closes the report:
# an open-only lookup would forget that identity and the next failing
# run would open a duplicate. The oldest match stays the canonical
# report and is reopened below when a problem recurs.
run: |
match=$(gh issue list --repo "$GITHUB_REPOSITORY" --state all \
--search "in:title \"$REPORT_TITLE\" author:app/github-actions" \
--limit 50 --json number,title,state \
--jq "[.[] | select(.title == \"$REPORT_TITLE\")] | sort_by(.number) | first // empty")
echo "number=$(jq -r '.number // empty' <<<"$match")" >> "$GITHUB_OUTPUT"
echo "state=$(jq -r '.state // empty' <<<"$match")" >> "$GITHUB_OUTPUT"
- name: Download report
if: env.STATUS == 'findings'
uses: actions/download-artifact@v8
with:
name: link-report
path: ./lychee
- name: Compose report
if: env.STATUS == 'findings'
run: |
run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
{
echo "Broken documentation links found by [\`$GITHUB_WORKFLOW\`]($run_url)."
echo
echo "This issue is rewritten by every scheduled run and closed automatically once all links resolve."
echo
echo "Entries can be false positives: some sites rate-limit or block automated clients while working fine in a browser. Confirm before editing the docs, and add persistent offenders to \`--exclude\` in \`.github/workflows/docs-link-check.yml\`."
echo
# Timeouts are reported alongside errors: entries land in
# timeout_map with a status text instead of an HTTP code.
jq -r '
"\(.errors) of \(.total) links failed, \(.timeouts) timed out.",
"",
([(.error_map | to_entries[]), (.timeout_map | to_entries[])]
| group_by(.key)[] |
"### Errors in \(.[0].key)",
"",
(map(.value[])[] | "* [\(.status.code // .status.text // "ERR")] <\(.url)> — \(.status.details // .status.text // "unknown error")"),
"")
' ./lychee/out.json
} > ./lychee/issue.md
- name: Compose checker error report
if: env.STATUS == 'checker-error'
run: |
mkdir -p ./lychee
run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
{
echo "The documentation link check did not complete in [the latest run]($run_url)."
echo
echo "This issue is rewritten by every scheduled run and closed automatically once a trustworthy run finds that all links resolve."
echo
echo "The checker did not produce a trustworthy link verdict. Treat the previous result, if any, as stale until a later run completes."
echo
echo "* Action outcome: \`$CHECKER_OUTCOME\`"
echo "* Exit code: \`${EXIT_CODE:-not reported}\`"
echo "* Verdict validation: \`failed\`"
} > ./lychee/issue.md
- name: Reopen report issue
# A healthy run closes the report, and the issue action below only
# rewrites the body of whatever number it is given. Without an
# explicit reopen, a later finding or checker error would rewrite a
# closed issue. A CLOSED state implies the lookup found a canonical
# issue, so no separate emptiness check.
if: >-
env.STATUS != 'healthy' &&
steps.report.outputs.state == 'CLOSED'
env:
ISSUE_NUMBER: ${{ steps.report.outputs.number }}
run: |
run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
gh issue reopen "$ISSUE_NUMBER" --repo "$GITHUB_REPOSITORY" \
--comment "The documentation link checker reported a problem again in [the latest run]($run_url)."
- name: Report link-check problem
if: env.STATUS != 'healthy'
uses: peter-evans/create-issue-from-file@fca9117c27cdc29c6c4db3b86c48e4115a786710 # v6.0.0
with:
# Empty on the first failing run, which creates the issue; afterwards
# the same issue is updated in place.
issue-number: ${{ steps.report.outputs.number }}
title: ${{ env.REPORT_TITLE }}
content-filepath: ./lychee/issue.md
labels: documentation
- name: Close report issue once links are healthy
# An OPEN state implies the lookup found a canonical issue; a report
# that is already closed needs nothing.
if: >-
env.STATUS == 'healthy' &&
steps.report.outputs.state == 'OPEN'
env:
ISSUE_NUMBER: ${{ steps.report.outputs.number }}
run: |
run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
gh issue close "$ISSUE_NUMBER" --repo "$GITHUB_REPOSITORY" \
--comment "All documentation links resolved in [the latest run]($run_url)."
.github/workflows/docs.yml
+1 -3
View File
@@ -55,9 +55,7 @@ jobs:
- name: Set up node
uses: actions/setup-node@v6
with:
node-version: 20
cache: 'npm'
cache-dependency-path: docs/package-lock.json
node-version: 24
- name: Install node dependencies
working-directory: nodejs
run: |
.github/workflows/nodejs.yml
+77 -14
View File
@@ -47,9 +47,8 @@ jobs:
version: 11.1.1
- uses: actions/setup-node@v6
with:
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
# in October. The library itself still supports Node >= 18
# (see test matrix below).
# Build on a supported LTS; the matrix job below covers every
# Node version the library claims to support.
node-version: 24
cache: 'pnpm'
cache-dependency-path: nodejs/pnpm-lock.yaml
@@ -84,7 +83,7 @@ jobs:
timeout-minutes: 30
strategy:
matrix:
node-version: [ "18", "20" ]
node-version: [ "22", "24", "26" ]
runs-on: "ubuntu-22.04"
defaults:
run:
@@ -101,9 +100,9 @@ jobs:
- uses: actions/setup-node@v6
name: Setup Node.js 24 for build
with:
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
# in October. Build/install runs on Node 24; tests run on the
# matrix version below using direct jest invocation.
# Build and install once on a fixed version so the generated docs
# are identical across matrix legs; the tests below then run on each
# supported Node version.
node-version: 24
cache: 'pnpm'
cache-dependency-path: nodejs/pnpm-lock.yaml
@@ -152,9 +151,9 @@ jobs:
S3_TEST: "1"
# Newer @smithy/core uses dynamic ESM imports.
NODE_OPTIONS: "--experimental-vm-modules"
# Invoke jest directly because pnpm 11 itself requires Node 22+
# while the matrix tests on older Node versions.
run: npx jest --verbose
# Invoke the installed jest binary directly; the pnpm shim is set up
# against the build-phase Node, not the version selected above.
run: node_modules/.bin/jest --verbose
- name: Test examples
working-directory: ./
env:
@@ -164,7 +163,72 @@ jobs:
run: |
python ci/mock_openai.py &
cd nodejs/examples
npx jest --testEnvironment jest-environment-node-single-context --verbose
node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose
arrow-compat:
# The main jobs resolve `apache-arrow` to the version pinned in
# devDependencies (the newest supported release). This job re-runs the
# suite with the library itself running on each older Arrow release the
# peer range still accepts, which is what a consumer on that version gets.
name: Linux (Arrow ${{ matrix.arrow-version }})
timeout-minutes: 30
strategy:
matrix:
arrow-version: [ "19.0.1", "20.0.0" ]
runs-on: "ubuntu-22.04"
defaults:
run:
shell: bash
working-directory: nodejs
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
lfs: true
- uses: pnpm/action-setup@v6
with:
version: 11.1.1
- uses: actions/setup-node@v6
with:
node-version: 24
cache: 'pnpm'
cache-dependency-path: nodejs/pnpm-lock.yaml
- uses: Swatinem/rust-cache@v2
with:
# Same crate, profile and target as the `linux` job above, so reuse
# its cache: rust-cache keys on the job id by default, and this job
# would otherwise start cold on every run until it had saved its own.
shared-key: linux
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Install dependencies
run: |
sudo apt update
sudo apt install -y protobuf-compiler libssl-dev
- name: Build
run: |
pnpm install --frozen-lockfile
# No `--` separator: pnpm forwards it literally, which would
# make napi-rs treat `--profile ci` as a cargo passthrough arg.
pnpm build:debug --profile ci
- name: Resolve apache-arrow to ${{ matrix.arrow-version }}
run: |
# The older releases are already installed as the `apache-arrow-NN`
# test aliases. With pnpm's hoisted linker each alias is a complete,
# self-contained directory, so swap it in for the library's own
# `apache-arrow` instead of running a second install, which has been
# seen to hang after printing "Done" in CI. A copy, not a symlink: the
# cross-version tests rely on the alias being a separate module
# instance from the one the library loads.
major="$(echo '${{ matrix.arrow-version }}' | cut -d. -f1)"
mv node_modules/apache-arrow node_modules/apache-arrow.pinned
cp -R "node_modules/apache-arrow-${major}" node_modules/apache-arrow
# `require("apache-arrow/package.json")` is blocked by the package's exports map.
node -p "'apache-arrow resolved to ' + JSON.parse(require('fs').readFileSync('node_modules/apache-arrow/package.json')).version"
pnpm tsc
- name: Test
env:
# Newer @smithy/core uses dynamic ESM imports.
NODE_OPTIONS: "--experimental-vm-modules"
run: node_modules/.bin/jest --verbose
macos:
timeout-minutes: 30
# macos-15 ships a newer linker; the older macos-14 linker fails to insert
@@ -185,8 +249,7 @@ jobs:
version: 11.1.1
- uses: actions/setup-node@v6
with:
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
# in October.
# pnpm 11 requires Node >= 22.13.
node-version: 24
cache: 'pnpm'
cache-dependency-path: nodejs/pnpm-lock.yaml
@@ -209,4 +272,4 @@ jobs:
pnpm tsc
- name: Test
run: |
pnpm test
pnpm test --runInBand
.github/workflows/npm-publish.yml
+81 -36
View File
@@ -40,40 +40,31 @@ jobs:
- target: aarch64-apple-darwin
host: macos-latest
features: fp16kernels
# Fat LTO was ~111 of this job's ~113 minutes.
lto: thin
codegen_units: 16
pre_build: |-
brew install protobuf
# Fat LTO (the workspace default in .cargo/config.toml) is
# single-threaded and is the peak-memory step of the build. On
# this runner it accounted for ~111 of the job's ~113 minutes,
# making it the critical path of the entire publish pipeline.
# ThinLTO parallelizes it across the runner's cores, for a few
# percent of runtime performance.
export CARGO_PROFILE_RELEASE_LTO=thin
export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
- target: x86_64-pc-windows-msvc
host: windows-2025
features: ","
# The lower peak also keeps this on the standard 4-core runner.
lto: thin
codegen_units: 16
pre_build: |-
choco install --no-progress protoc ninja nasm
tail -n 1000 /c/ProgramData/chocolatey/logs/chocolatey.log
# There is an issue where choco doesn't add nasm to the path
export PATH="$PATH:/c/Program Files/NASM"
nasm -v
# See the ThinLTO note on aarch64-apple-darwin above. Keeping
# peak memory down is also what lets this run on the standard
# 4-core runner: the 8-core larger runner was only needed to
# stop fat LTO from OOMing rustc-LLVM.
export CARGO_PROFILE_RELEASE_LTO=thin
export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
- target: aarch64-pc-windows-msvc
host: windows-2025
features: ","
lto: thin
codegen_units: 16
pre_build: |-
choco install --no-progress protoc
rustup target add aarch64-pc-windows-msvc
# See the ThinLTO note on aarch64-apple-darwin above.
export CARGO_PROFILE_RELEASE_LTO=thin
export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
- target: x86_64-unknown-linux-gnu
host: ubuntu-latest
features: fp16kernels
@@ -103,6 +94,14 @@ jobs:
# https://github.com/napi-rs/napi-rs/blob/main/debian-aarch64.Dockerfile
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian-aarch64
features: "fp16kernels"
# Fat LTO OOM-killed rustc every nightly; even with lld it peaked
# at 31391 MiB of the runner's 32 GiB.
lto: thin
codegen_units: 16
# arm64 Linux links through GNU `ld` where x86_64 defaults to
# `rust-lld`, which is why only arm64 OOM'd. lld cut the largest
# linker process 7.0 -> 4.0 GiB (lancedb/sophon#7313).
linker: /tmp/aarch64-lld-clang
pre_build: |-
set -e &&
apt-get update &&
@@ -112,9 +111,30 @@ jobs:
# AT_HWCAP2 (added in Linux 3.17). Define it for aws-lc-sys.
export CFLAGS="$CFLAGS -DAT_HWCAP2=26" &&
rustup target add aarch64-unknown-linux-gnu
# Not `&&`-chained: in dash, errexit does not fire for a
# non-final command in an `&&` list, so failures were ignored.
#
# A wrapper rather than `-C link-arg` because the per-target
# rustflags variable does not reach every unit that links, while
# the linker variable does. `clang` because GCC silently ignores
# `-fuse-ld=lld` unless built with lld support. Two echoes
# because printf's newline escape gets rewritten to `;` between
# here and the container.
echo '#!/bin/sh' > /tmp/aarch64-lld-clang
echo 'exec clang --target=aarch64-unknown-linux-gnu --sysroot=/usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot --gcc-toolchain=/usr/aarch64-unknown-linux-gnu -fuse-ld=lld "$@"' >> /tmp/aarch64-lld-clang
chmod 0755 /tmp/aarch64-lld-clang
# Fail now, not at the cdylib link ~30 minutes later. Linking at
# all also proves lld resolved; clang errors out when it cannot.
echo 'int main(void){return 0;}' > /tmp/probe.c
/tmp/aarch64-lld-clang /tmp/probe.c -o /tmp/probe
readelf -h /tmp/probe | grep AArch64
- target: aarch64-unknown-linux-musl
host: ubuntu-2404-8x-x64
features: ","
# Fat LTO took the whole runner down. lld cannot help: it died
# inside rustc's LLVM, before any linker was spawned.
lto: thin
codegen_units: 16
pre_build: |-
set -e &&
sudo apt-get update &&
@@ -123,6 +143,19 @@ jobs:
export EXTRA_ARGS="-x"
name: build - ${{ matrix.settings.target }}
runs-on: ${{ matrix.settings.host }}
# On the job, not exported from `pre_build`: `Swatinem/rust-cache` hashes
# `CARGO_*` into its cache key before any step runs, so a step-local export
# leaves the key unchanged while cargo still rebuilds cold. The ThinLTO
# legs had been doing that every run.
#
# Not `RUSTFLAGS`: setting it, even to "", discards every config-file
# rustflag, silently dropping .cargo/config.toml's `target-cpu` and
# `target-feature` from the published binaries.
env:
CARGO_PROFILE_RELEASE_LTO: ${{ matrix.settings.lto || 'fat' }}
CARGO_PROFILE_RELEASE_CODEGEN_UNITS: ${{ matrix.settings.codegen_units || '1' }}
# Empty elsewhere: a per-target variable is only read for that triple.
CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: ${{ matrix.settings.linker }}
defaults:
run:
working-directory: nodejs
@@ -135,8 +168,7 @@ jobs:
- name: Setup node
uses: actions/setup-node@v6
with:
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
# in October.
# pnpm 11 requires Node >= 22.13.
node-version: 24
cache: pnpm
cache-dependency-path: nodejs/pnpm-lock.yaml
@@ -169,19 +201,15 @@ jobs:
# creating ref). The nightly cadence also keeps entries inside
# GitHub's 7-day eviction window, which a tag-only trigger would not.
save-if: ${{ github.ref == 'refs/heads/main' }}
# Docker builds can use rust-cache too. `target/` already lives on the
# host because the whole workspace is bind-mounted into the container, and
# rust-cache's prune and save run host-side, so they can manage it -- which
# is what keeps the entry to dependency artifacts rather than a multi-GB
# copy of everything.
# Docker builds can use rust-cache too: the workspace is bind-mounted, so
# `target/` lives on the host and rust-cache's prune keeps the entry
# small.
#
# Two differences from the native builds. The container's CARGO_HOME is
# bind-mounted from `.cargo-cache` rather than the host's ~/.cargo, so that
# has to be cached explicitly. And the key is derived from the *host* rustc
# version, which is not the compiler that produced these artifacts; that is
# safe because cargo fingerprints the real compiler and rebuilds on a
# mismatch, it just means a base-image toolchain bump costs one cold build
# instead of invalidating the key.
# bind-mounted from `.cargo-cache` rather than ~/.cargo, so that is cached
# explicitly. And the key uses the *host* rustc version, not the compiler
# that built these artifacts -- safe, since cargo fingerprints the real
# one; a base-image bump just costs one cold build.
- name: Cache cargo (docker builds)
uses: Swatinem/rust-cache@v2
if: ${{ matrix.settings.docker }}
@@ -210,14 +238,19 @@ jobs:
# cache step above saves. Previously the registry mounts pointed at
# `.cargo/...`, a path nothing cached, so the container re-downloaded
# the whole crate registry on every run.
#
# `docker run` inherits nothing; `-e NAME` carries the job's `env:` in.
options: "--user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db \
-v ${{ github.workspace }}/.cargo-cache/registry/cache:/usr/local/cargo/registry/cache \
-v ${{ github.workspace }}/.cargo-cache/registry/index:/usr/local/cargo/registry/index \
-e CARGO_PROFILE_RELEASE_LTO \
-e CARGO_PROFILE_RELEASE_CODEGEN_UNITS \
-e CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER \
-v ${{ github.workspace }}:/build -w /build/nodejs"
run: |
set -e
${{ matrix.settings.pre_build }}
npx napi build --platform --release \
node_modules/.bin/napi build --platform --release \
--features ${{ matrix.settings.features }} \
--target ${{ matrix.settings.target }} \
--dts ../lancedb/native.d.ts \
@@ -237,7 +270,7 @@ jobs:
- name: Build
run: |
${{ matrix.settings.pre_build }}
npx napi build --platform --release \
node_modules/.bin/napi build --platform --release \
--features ${{ matrix.settings.features }} \
--target ${{ matrix.settings.target }} \
--dts ../lancedb/native.d.ts \
@@ -256,6 +289,18 @@ jobs:
if: always()
run: df -h
shell: bash
- name: Report peak memory
if: always() && runner.os == 'Linux'
shell: bash
run: |
peak=$(find /sys/fs/cgroup -name memory.peak -readable \
-exec cat {} + 2>/dev/null | sort -n | tail -1)
if [ -n "$peak" ]; then
echo "peak memory: $((peak / 1024 / 1024)) MiB"
else
echo "peak memory: unavailable (no readable cgroup v2 memory.peak)"
fi
free -g || true
- name: Upload artifact
uses: actions/upload-artifact@v7
with:
@@ -293,7 +338,7 @@ jobs:
- target: aarch64-unknown-linux-gnu
host: ubuntu-2404-8x-arm64
node:
- '20'
- '22'
runs-on: ${{ matrix.settings.host }}
defaults:
run:
@@ -339,9 +384,9 @@ jobs:
- name: Move built files
run: cp dist/native.d.ts dist/native.js dist/*.node lancedb/
- name: Test bindings
# Invoke jest directly because pnpm 11 itself requires Node 22+
# while the matrix tests on older Node versions.
run: npx jest --verbose
# Invoke the installed jest binary directly; the pnpm shim is set up
# against the install-phase Node, not the version selected above.
run: node_modules/.bin/jest --verbose
publish:
name: Publish
runs-on: ubuntu-latest
.github/workflows/pypi-publish.yml
+19 -4
View File
@@ -21,7 +21,7 @@ permissions:
contents: read
# Without this, a force-push to a PR leaves the previous run going -- including
# a ~74 minute Windows job and a billed arm64 wheel build.
# a long fat-LTO Windows job and a billed arm64 wheel build.
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
@@ -69,6 +69,16 @@ jobs:
uses: actions/setup-python@v6
with:
python-version: "3.10"
- name: Add swap for Arm fat LTO
if: matrix.config.platform == 'aarch64'
shell: bash
run: |
swap_file="$RUNNER_TEMP/lancedb-swap"
sudo fallocate --length 16G "$swap_file"
sudo chmod 600 "$swap_file"
sudo mkswap "$swap_file"
sudo swapon "$swap_file"
free -h
- uses: ./.github/workflows/build_linux_wheel
with:
python-minor-version: 10
@@ -113,12 +123,17 @@ jobs:
path: target/wheels/lancedb-*.whl
if-no-files-found: error
windows:
timeout-minutes: 90
runs-on: windows-latest
# The fat-LTO build of the `lancedb` crate does not finish within 90 minutes
# on the 4-core standard runner, so use the 8-core larger runner with extra
# headroom.
timeout-minutes: 150
runs-on: windows-2025-8x-x64
env:
# link.exe is single-threaded and the long pole on Windows builds. Use
# rustc's bundled lld-link instead.
CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER: rust-lld
# Keep the repository's fat LTO / 1 codegen unit release profile to leave
# headroom below PyPI's file-size limit for Windows wheels.
steps:
- uses: actions/checkout@v6
with:
@@ -133,7 +148,7 @@ jobs:
# or the default branch -- so with no run on main there is nothing that
# can populate an entry the release build would be allowed to read. Fixing
# this needs a main/nightly trigger (which would also catch wheel-build
# breakage before a release); the ~74 minutes here is otherwise dominated
# breakage before a release); the build time here is otherwise dominated
# by the fat-LTO link, which no cache avoids.
- uses: ./.github/workflows/build_windows_wheel
with:
.github/workflows/python.yml
+4 -4
View File
@@ -156,7 +156,7 @@ jobs:
- name: Test without pylance or pandas
run: |
pip uninstall -y pylance pandas
pytest -vv python/tests/test_table.py
pytest -vv python/tests/test_table.py python/tests/test_namespace_no_pylance.py
# Make sure wheels are not included in the Rust cache
- name: Delete wheels
run: rm -rf target/wheels
@@ -229,7 +229,8 @@ jobs:
# Make sure wheels are not included in the Rust cache
- name: Delete wheels
run: rm -rf target/wheels
pydantic1x:
min-deps:
name: "Minimum dependencies"
timeout-minutes: 60
runs-on: "ubuntu-24.04"
defaults:
@@ -259,8 +260,7 @@ jobs:
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Install lancedb
run: |
pip install "pydantic<2"
pip install pyarrow==16
pip install "pydantic==2.7.4" "pyarrow==16"
pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests]
- name: Run tests
run: pytest -m "not slow and not s3_test" -x -v --durations=30 python/tests
.github/workflows/rust.yml
+46 -12
View File
@@ -121,7 +121,6 @@ jobs:
# Need up-to-date compilers for kernels
CC: clang-18
CXX: clang++-18
GH_TOKEN: ${{ secrets.SOPHON_READ_TOKEN }}
steps:
- uses: actions/checkout@v6
with:
@@ -165,11 +164,41 @@ jobs:
- name: Run feature tests
run: CARGO_ARGS="--profile ci" make -C ./lancedb feature-tests
- name: Run examples
run: cargo run --profile ci --example simple --locked
run: cargo run --profile ci --all-features --example simple --locked
remote:
timeout-minutes: 30
# Running this requires access to secrets, so skip if this is a PR from a
# fork. Keep it separate from the all-features build so Cargo does not
# retain both dependency graphs in one target directory.
if: github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork
runs-on: ubuntu-2404-4x-x64
defaults:
run:
shell: bash
working-directory: rust
env:
CC: clang-18
CXX: clang++-18
GH_TOKEN: ${{ github.token }}
SOPHON_READ_TOKEN: ${{ secrets.SOPHON_READ_TOKEN }}
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
lfs: true
- uses: Swatinem/rust-cache@v2
with:
# Remote tests use a different feature graph from the main Linux
# job. Cache downloads, but build into a fresh target directory.
cache-targets: false
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Install dependencies
run: |
sudo apt update
sudo apt install -y protobuf-compiler libssl-dev
- uses: rui314/setup-mold@v1
- name: Run remote tests
# Running this requires access to secrets, so skip if this is
# a PR from a fork.
if: github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork
run: CARGO_ARGS="--profile ci" make -C ./lancedb remote-tests
macos:
@@ -204,7 +233,10 @@ jobs:
ALL_FEATURES=`cargo metadata --format-version=1 --no-deps \
| jq -r '.packages[] | .features | keys | .[]' \
| grep -v s3-test | sort | uniq | paste -s -d "," -`
cargo test --profile ci --features $ALL_FEATURES --locked
# Run doctests before test binaries fill the runner disk. Examples are
# already built by the Linux job, so avoid retaining them here.
cargo test --profile ci --features $ALL_FEATURES --locked --doc
cargo test --profile ci --features $ALL_FEATURES --locked --lib --tests
windows:
strategy:
@@ -296,16 +328,18 @@ jobs:
cargo update -p aws-types --precise 1.3.9
cargo update -p aws-sigv4 --precise 1.3.5
cargo update -p aws-credential-types --precise 1.2.8
cargo update -p aws-smithy-checksums --precise 0.63.9
# aws-smithy-checksums must stay at or above 0.63.13: OpenDAL's S3
# service needs crc-fast ~1.9, and older releases pin it to ~1.3.
cargo update -p aws-smithy-checksums --precise 0.63.13
cargo update -p aws-smithy-runtime --precise 1.9.3
cargo update -p aws-smithy-http --precise 0.62.4
cargo update -p aws-smithy-eventstream --precise 0.60.12
cargo update -p aws-smithy-http --precise 0.62.6
cargo update -p aws-smithy-eventstream --precise 0.60.14
cargo update -p aws-smithy-http-client --precise 1.1.3
cargo update -p aws-smithy-observability --precise 0.1.4
cargo update -p aws-smithy-query --precise 0.60.8
cargo update -p aws-smithy-runtime-api --precise 1.9.1
cargo update -p aws-smithy-async --precise 1.2.6
cargo update -p aws-smithy-types --precise 1.3.5
cargo update -p aws-smithy-runtime-api --precise 1.9.3
cargo update -p aws-smithy-async --precise 1.2.7
cargo update -p aws-smithy-types --precise 1.3.6
cargo update -p aws-smithy-xml --precise 0.60.11
cargo update -p home --precise 0.5.9
- name: cargo +${{ matrix.msrv }} check
.github/workflows/typos.yml
+20
View File
@@ -0,0 +1,20 @@
name: Typo checker
on:
push:
branches:
- main
pull_request:
permissions:
contents: read
jobs:
run:
name: Spell Check with Typos
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Check spelling of the entire repository
uses: crate-ci/typos@6802cc60d4e7f78b9d5454f6cf3935c042d5e1e3 # v1.26.0
.github/workflows/update_package_lock_run.yml
-22
View File
@@ -1,22 +0,0 @@
name: Update package-lock.json
on:
workflow_dispatch:
permissions:
contents: read
jobs:
publish:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
with:
ref: main
persist-credentials: false
fetch-depth: 0
lfs: true
- uses: ./.github/workflows/update_package_lock
with:
github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
.github/workflows/update_package_lock_run_nodejs.yml
-22
View File
@@ -1,22 +0,0 @@
name: Update NodeJs package-lock.json
on:
workflow_dispatch:
permissions:
contents: read
jobs:
publish:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
with:
ref: main
persist-credentials: false
fetch-depth: 0
lfs: true
- uses: ./.github/workflows/update_package_lock_nodejs
with:
github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
.pre-commit-config.yaml
+8 -1
View File
@@ -10,6 +10,10 @@ repos:
rev: v0.9.9
hooks:
- id: ruff
- repo: https://github.com/crate-ci/typos
rev: v1.26.0
hooks:
- id: typos
# - repo: https://github.com/RobertCraigie/pyright-python
# rev: v1.1.395
# hooks:
@@ -20,7 +24,10 @@ repos:
hooks:
- id: local-biome-check
name: biome check
entry: npx @biomejs/biome@1.8.3 check --config-path nodejs/biome.json nodejs/
# Use the biome from nodejs/package.json rather than a separately
# pinned one: the two drifted apart and disagreed on formatting, so
# this hook rejected code that `pnpm lint` accepted.
entry: nodejs/node_modules/.bin/biome check --config-path nodejs/biome.json nodejs/
language: system
types: [text]
files: "nodejs/.*"
.typos.toml
+19
View File
@@ -0,0 +1,19 @@
[default]
extend-ignore-re = ["(?Rm)^.*(#|//)\\s*spellchecker:disable-line$"]
[default.extend-words]
# Azure Kubernetes Service, mentioned in rust/lancedb/src/remote/oauth.rs.
AKS = "AKS"
# RabitQ is the name of a vector quantization algorithm, not a typo of "Rabbit".
Rabit = "Rabit"
# `VarBuilder::from_mmaped_safetensors` is the real (if oddly-spelled) name of
# the candle-core API we call in rust/lancedb/src/embeddings/sentence_transformers.rs.
mmaped = "mmaped"
# `WriteableBuffer` is the real name of a type from Python's `_typeshed` stubs,
# used in python/python/lancedb/_blob.py.
Writeable = "Writeable"
[files]
extend-exclude = [
"*_THIRD_PARTY_LICENSES.*",
]
AGENTS.md
+6 -3
View File
@@ -18,6 +18,9 @@ Common commands:
* Run specific test: `cargo test --quiet --features remote -p <package_name> --test <test_name>`
* Lint: `cargo clippy --quiet --features remote --tests --examples`
* Format Rust: `cargo fmt --all`
* Use repository-defined Cargo profiles instead of ad hoc LTO overrides.
* Use `release-with-debug` for benchmarks and profiling so optimized builds keep debug symbols without a rebuild.
* Use `release-no-lto` only for local debugging, IO-bound benchmarks, or compile-time-sensitive performance investigation where LTO would not affect the measured bottleneck.
* Format Python: `ruff format .`
* Lint Python: `ruff check .`
* Bootstrap Python dev env: `cd python && uv run --extra tests --extra dev maturin develop --extras tests,dev`
@@ -35,7 +38,7 @@ Before committing changes, run formatting for every language you touched. At min
* Rust changes: run `cargo fmt --all`.
* Python changes: run `ruff format .` and `ruff check .` from the repository root,
and run targeted tests through `cd python && uv run ...`.
* TypeScript changes: run the relevant `npm`/`pnpm` lint, format, build, and docs commands in `nodejs`.
* TypeScript changes: run the relevant `pnpm` lint, format, build, and docs commands in `nodejs`.
Before creating a PR, the exact value passed to `gh pr create --title` must follow
Conventional Commits, such as `fix: support nested field paths in native index creation`
@@ -98,12 +101,12 @@ Python bindings changes:
TypeScript bindings changes:
1. Add napi-rs method binding on `Table` in `nodejs/src/table.rs`.
2. Run `npm run build` to generate TypeScript definitions.
2. Run `pnpm build` to generate TypeScript definitions.
3. Add typescript method on abstract class `Table` in `nodejs/src/table.ts`.
4. Add concrete method on `LocalTable` class in `nodejs/src/native_table.ts`.
* Note: despite the name, this class is also used for remote tables.
5. Add test in `nodejs/__test__/table.test.ts`.
6. Run `npm run docs` to generate TypeScript documentation.
6. Run `pnpm run docs` to generate TypeScript documentation.
## Python API reference
CLAUDE.md
-1
View File
@@ -1 +0,0 @@
AGENTS.md
Cargo.lock
Generated
+1073 -473
View File
File diff suppressed because it is too large Load Diff
Cargo.toml
+29 -17
View File
@@ -13,20 +13,21 @@ categories = ["database-implementations"]
rust-version = "1.91.0"
[workspace.dependencies]
lance = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=13.1.0-beta.1", default-features = false, "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=13.1.0-beta.1", default-features = false, "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=13.1.0-beta.1", default-features = false, "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lancedb = { path = "rust/lancedb", default-features = false }
ahash = "0.8"
# Note that this one does not include pyarrow
arrow = { version = "58.0.0", optional = false }
@@ -38,7 +39,11 @@ arrow-ord = "58.0.0"
arrow-schema = "58.0.0"
arrow-select = "58.0.0"
arrow-cast = "58.0.0"
arrow-flight = { version = "58.0.0", features = ["flight-sql-experimental"] }
async-trait = "0"
# Smithy JSON 0.63 requires the pre-1.7 Document representation; allow the MSRV pin.
aws-smithy-types = ">=1.3.6, <1.7"
bytes = "1"
datafusion = { version = "54.0.0", default-features = false }
datafusion-catalog = "54.0.0"
datafusion-common = { version = "54.0.0", default-features = false }
@@ -52,20 +57,27 @@ env_logger = "0.11"
half = { "version" = "2.7.1", default-features = false, features = [
"num-traits",
] }
futures = "0"
futures = "0.3"
log = "0.4"
metrics = "0.24"
metrics-util = "0.19"
moka = { version = "0.12", features = ["future"] }
object_store = "0.13.2"
object_store = "0.14.1"
pin-project = "1.0.7"
rand = "0.9"
snafu = "0.8"
url = "2"
num-traits = "0.2"
oauth2 = { version = "5.0", default-features = false }
regex = "1.10"
semver = "1.0.25"
chrono = "0.4"
serde = "1"
serde_json = "1"
tempfile = "3.5.0"
tokio = { version = "1.23", features = ["rt-multi-thread", "sync"] }
tonic = { version = "0.14", features = ["tls-native-roots", "tls-ring"] }
uuid = { version = "1.7.0", features = ["v4", "v7"] }
chrono = { version = "0.4", default-features = false, features = ["clock"] }
[profile.ci]
debug = "line-tables-only"
Makefile
+1 -1
View File
@@ -5,5 +5,5 @@ licenses:
cd python && cargo about generate ../about.hbs -o RUST_THIRD_PARTY_LICENSES.html -c ../about.toml
cd python && uv sync --all-extras && uv tool run pip-licenses --python .venv/bin/python --format=markdown --with-urls --output-file=PYTHON_THIRD_PARTY_LICENSES.md
cd nodejs && cargo about generate ../about.hbs -o RUST_THIRD_PARTY_LICENSES.html -c ../about.toml
cd nodejs && npx license-checker --markdown --out NODEJS_THIRD_PARTY_LICENSES.md
cd nodejs && pnpm dlx license-checker@25 --markdown --out NODEJS_THIRD_PARTY_LICENSES.md
cd java && ./mvnw license:aggregate-add-third-party -q
ci/check_breaking_changes.py
+2 -1
View File
@@ -2,6 +2,7 @@
Check whether there are any breaking changes in the PRs between the base and head commits.
If there are, assert that we have incremented the minor version.
"""
import argparse
import os
from packaging.version import parse
@@ -27,7 +28,7 @@ if __name__ == "__main__":
else:
print("No breaking changes found.")
exit(0)
last_stable_version = parse(args.last_stable_version)
current_version = parse(args.current_version)
if current_version.minor <= last_stable_version.minor:
ci/check_lance_release.py
+14 -3
View File
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
"""Determine whether a newer Lance tag exists and expose results for CI."""
from __future__ import annotations
import argparse
@@ -36,8 +37,16 @@ class SemVer:
prerelease: Tuple[Union[int, str], ...]
def __lt__(self, other: "SemVer") -> bool: # pragma: no cover - simple comparison
if (self.major, self.minor, self.patch) != (other.major, other.minor, other.patch):
return (self.major, self.minor, self.patch) < (other.major, other.minor, other.patch)
if (self.major, self.minor, self.patch) != (
other.major,
other.minor,
other.patch,
):
return (self.major, self.minor, self.patch) < (
other.major,
other.minor,
other.patch,
)
if self.prerelease == other.prerelease:
return False
if not self.prerelease:
@@ -142,7 +151,9 @@ def read_current_version(repo_root: Path) -> str:
deps = data["workspace"]["dependencies"]
entry = deps["lance"]
except KeyError as exc: # pragma: no cover - configuration guard
raise RuntimeError("Failed to locate workspace.dependencies.lance in Cargo.toml") from exc
raise RuntimeError(
"Failed to locate workspace.dependencies.lance in Cargo.toml"
) from exc
if isinstance(entry, str):
raw_version = entry
ci/mock_openai.py
+9 -6
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""A zero-dependency mock OpenAI embeddings API endpoint for testing purposes."""
import argparse
import json
import http.server
@@ -22,11 +23,13 @@ class MockOpenAIRequestHandler(http.server.BaseHTTPRequestHandler):
data = []
for i in range(num_inputs):
data.append({
"object": "embedding",
"embedding": [0.1] * 1536,
"index": i,
})
data.append(
{
"object": "embedding",
"embedding": [0.1] * 1536,
"index": i,
}
)
response = {
"object": "list",
@@ -35,7 +38,7 @@ class MockOpenAIRequestHandler(http.server.BaseHTTPRequestHandler):
"usage": {
"prompt_tokens": 0,
"total_tokens": 0,
}
},
}
self.send_response(200)
ci/semver_sort.py
+1
View File
@@ -7,6 +7,7 @@ from packaging.version import parse, InvalidVersion
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("prefix", default="v")
args = parser.parse_args()
ci/set_lance_version.py
+4 -4
View File
@@ -22,7 +22,7 @@ def run_command(command: str) -> str:
def get_latest_stable_version() -> str:
version_line = run_command("cargo info lance | grep '^version:'")
# Example output: "version: 0.35.0 (latest 0.37.0)"
match = re.search(r'\(latest ([0-9.]+)\)', version_line)
match = re.search(r"\(latest ([0-9.]+)\)", version_line)
if match:
return match.group(1)
# Fallback: use the first version after 'version:'
@@ -69,7 +69,7 @@ def extract_default_features(line: str) -> bool:
"""
import re
match = re.search(r'default-features\s*=\s*false', line)
match = re.search(r"default-features\s*=\s*false", line)
return match is not None
@@ -104,7 +104,7 @@ def dict_to_toml_line(package_name: str, config: dict) -> str:
# This shouldn't happen with our current usage
parts.append(f'"{key}" = {json.dumps(value)}')
return f'{package_name} = {{ {", ".join(parts)} }}\n'
return f"{package_name} = {{ {', '.join(parts)} }}\n"
def update_cargo_toml(line_updater):
@@ -119,7 +119,7 @@ def update_cargo_toml(line_updater):
lance_line = ""
is_parsing_lance_line = False
for line in lines:
if line.startswith("lance"):
if re.match(r"^lance(?:\s|[-_])", line):
# Check if this is a single-line or multi-line entry
# Single-line entries either:
# 1. End with } (complete inline table)
ci/tests/test_set_lance_version.py
+185
View File
@@ -0,0 +1,185 @@
import os
import stat
import subprocess
import sys
import tempfile
import textwrap
import unittest
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[2]
SCRIPT = REPO_ROOT / "ci" / "set_lance_version.py"
LANCE_GIT_URL = "https://github.com/lance-format/lance.git"
CARGO_TOML = """\
[workspace.dependencies]
lance = { "version" = "=1.0.0", default-features = false, "features" = ["dynamodb"] }
lance-core = "1.0.0"
lance_datafusion = {
"version" = "=1.0.0",
"features" = ["substrait"]
}
lancedb = { path = "rust/lancedb", default-features = false }
lancedb-common = { path = "rust/lancedb-common" }
lancewood = "1.0.0"
my-lance = "1.0.0"
"""
UNTOUCHED_DEPENDENCIES = """\
lancedb = { path = "rust/lancedb", default-features = false }
lancedb-common = { path = "rust/lancedb-common" }
lancewood = "1.0.0"
my-lance = "1.0.0"
"""
class SetLanceVersionTest(unittest.TestCase):
def test_supported_update_modes_only_rewrite_lance_dependencies(self):
cases = {
"stable": (
"""\
lance = { "version" = "=9.9.9", default-features = false, "features" = ["dynamodb"] }
lance-core = "=9.9.9"
lance_datafusion = { "version" = "=9.9.9", "features" = ["substrait"] }
""",
["cargo info lance", "cargo metadata"],
),
"preview": (
f"""\
lance = {{ "version" = "=10.0.0-beta.3", default-features = false, "features" = ["dynamodb"], "tag" = "v10.0.0-beta.3", "git" = "{LANCE_GIT_URL}" }}
lance-core = {{ "version" = "=10.0.0-beta.3", "tag" = "v10.0.0-beta.3", "git" = "{LANCE_GIT_URL}" }}
lance_datafusion = {{ "version" = "=10.0.0-beta.3", "features" = ["substrait"], "tag" = "v10.0.0-beta.3", "git" = "{LANCE_GIT_URL}" }}
""",
["git ls-remote --tags", "cargo metadata"],
),
"local": (
"""\
lance = { "path" = "../lance/rust/lance", default-features = false, "features" = ["dynamodb"] }
lance-core = { "path" = "../lance/rust/lance-core" }
lance_datafusion = { "path" = "../lance/rust/lance_datafusion", "features" = ["substrait"] }
""",
["cargo metadata"],
),
"v8.1.2": (
"""\
lance = { "version" = "=8.1.2", default-features = false, "features" = ["dynamodb"] }
lance-core = "=8.1.2"
lance_datafusion = { "version" = "=8.1.2", "features" = ["substrait"] }
""",
["cargo metadata"],
),
"v8.2.0-beta.4": (
f"""\
lance = {{ "version" = "=8.2.0-beta.4", default-features = false, "features" = ["dynamodb"], "tag" = "v8.2.0-beta.4", "git" = "{LANCE_GIT_URL}" }}
lance-core = {{ "version" = "=8.2.0-beta.4", "tag" = "v8.2.0-beta.4", "git" = "{LANCE_GIT_URL}" }}
lance_datafusion = {{ "version" = "=8.2.0-beta.4", "features" = ["substrait"], "tag" = "v8.2.0-beta.4", "git" = "{LANCE_GIT_URL}" }}
""",
["cargo metadata"],
),
}
for version, (updated_dependencies, expected_commands) in cases.items():
with self.subTest(version=version), tempfile.TemporaryDirectory() as tmp:
workdir = Path(tmp)
(workdir / "Cargo.toml").write_text(CARGO_TOML)
command_log = workdir / "commands.log"
fake_bin = workdir / "bin"
fake_bin.mkdir()
self._write_fake_executables(fake_bin)
self._write_fake_python_dependencies(workdir)
env = os.environ.copy()
env["PATH"] = os.pathsep.join([str(fake_bin), env["PATH"]])
env["FAKE_COMMAND_LOG"] = str(command_log)
env["PYTHONPATH"] = os.pathsep.join(
filter(None, [str(workdir), env.get("PYTHONPATH")])
)
result = subprocess.run(
[sys.executable, str(SCRIPT), version],
cwd=workdir,
env=env,
capture_output=True,
text=True,
timeout=10,
)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(
(workdir / "Cargo.toml").read_text(),
"[workspace.dependencies]\n"
+ updated_dependencies
+ UNTOUCHED_DEPENDENCIES,
)
commands = command_log.read_text().splitlines()
for command in expected_commands:
self.assertTrue(
any(line.startswith(command) for line in commands),
f"{command!r} not found in {commands!r}",
)
def _write_fake_executables(self, fake_bin: Path) -> None:
cargo = fake_bin / "cargo"
cargo.write_text(
textwrap.dedent(
"""\
#!/bin/sh
printf 'cargo %s\\n' "$*" >> "$FAKE_COMMAND_LOG"
case "$1" in
info)
printf '%s\\n' 'version: 8.8.8 (latest 9.9.9)'
;;
metadata)
;;
*)
exit 2
;;
esac
"""
)
)
cargo.chmod(cargo.stat().st_mode | stat.S_IXUSR)
git = fake_bin / "git"
git.write_text(
textwrap.dedent(
"""\
#!/bin/sh
printf 'git %s\\n' "$*" >> "$FAKE_COMMAND_LOG"
if [ "$1" != "ls-remote" ]; then
exit 2
fi
printf '%s\\n' \\
'111111 refs/tags/v9.9.9' \\
'222222 refs/tags/v10.0.0-beta.1' \\
'333333 refs/tags/v10.0.0-beta.3'
"""
)
)
git.chmod(git.stat().st_mode | stat.S_IXUSR)
def _write_fake_python_dependencies(self, workdir: Path) -> None:
packaging = workdir / "packaging"
packaging.mkdir()
(packaging / "__init__.py").write_text("")
(packaging / "version.py").write_text(
textwrap.dedent(
"""\
class Version:
def __init__(self, value):
release, _, prerelease = value.partition("-beta.")
self._key = (
tuple(int(part) for part in release.split(".")),
not prerelease,
int(prerelease or 0),
)
def __lt__(self, other):
return self._key < other._key
"""
)
)
if __name__ == "__main__":
unittest.main()
ci/update_lockfiles.sh
+2 -6
View File
@@ -12,16 +12,12 @@ done
# This updates the lockfile without building
cargo metadata --quiet > /dev/null
pushd nodejs || exit 1
npm install --package-lock-only --silent
popd
if git diff --quiet --exit-code; then
echo "No lockfile changes to commit; skipping amend."
elif $AMEND; then
git add Cargo.lock nodejs/package-lock.json
git add Cargo.lock
git commit --amend --no-edit
else
git add Cargo.lock nodejs/package-lock.json
git add Cargo.lock
git commit -m "Update lockfiles"
fi
ci/validate_stable_lance.py
+1 -1
View File
@@ -12,7 +12,7 @@ with open("Cargo.toml", "rb") as f:
elif isinstance(dep, dict):
# Version doesn't have the beta tag in it, so we instead look
# at the git tag.
version = dep.get('tag', dep.get('version'))
version = dep.get("tag", dep.get("version"))
else:
raise ValueError("Unexpected type for dependency: " + str(dep))
deny.toml
+19 -11
View File
@@ -101,6 +101,19 @@ ignore = [
# https://rustsec.org/advisories/RUSTSEC-2026-0195
{ id = "RUSTSEC-2026-0194", reason = "transitive via inferno/lance/opendal; XML from trusted cloud endpoints, not attacker-controlled" },
{ id = "RUSTSEC-2026-0195", reason = "transitive via inferno/lance/opendal; XML from trusted cloud endpoints, not attacker-controlled" },
# smartstring: unmaintained — the repository was archived by its author on
# 2026-05-03. Not a vulnerability. Reached only transitively through polars
# (polars-core/-io/-ops/-time/-utils); nothing in LanceDB depends on it directly.
# The advisory states no safe upgrade is available: upstream recommends
# compact_str/smol_str, so clearing this requires polars to migrate.
# https://rustsec.org/advisories/RUSTSEC-2026-0249
{ id = "RUSTSEC-2026-0249", reason = "smartstring unmaintained via polars; no fixed upstream release" },
# h2 0.3: empty DATA frames can be queued without limit. The patched
# h2 0.4 line is locked to 0.4.16, but no patched 0.3 release exists.
# The old copy is pulled in by aws-smithy's legacy hyper 0.14 client.
# https://rustsec.org/advisories/RUSTSEC-2026-0258
{ id = "RUSTSEC-2026-0258", reason = "h2 0.3 via legacy aws-smithy/hyper 0.14; no patched 0.3 release" },
]
# ---------------------------------------------------------------------------
@@ -118,18 +131,13 @@ allow = [
"BSD-3-Clause",
"ISC",
"Unicode-3.0",
"Unicode-DFS-2016",
"Zlib",
"CC0-1.0",
"MPL-2.0",
"BSL-1.0",
"OpenSSL",
# 0BSD ("BSD Zero Clause") is effectively public domain — no attribution
# required. Pulled in by `mock_instant`.
"0BSD",
# bzip2-1.0.6 is the permissive upstream bzip2 license (BSD-like). Pulled
# in by `libbz2-rs-sys`, the pure-Rust bzip2 implementation.
"bzip2-1.0.6",
# CDLA-Permissive-2.0 is a permissive data license used by `webpki-roots`
# for the Mozilla CA root bundle. Data-only, distribution-compatible.
"CDLA-Permissive-2.0",
@@ -137,12 +145,7 @@ allow = [
confidence-threshold = 0.8
# Per-crate license exceptions: allow a license for a specific crate only,
# rather than globally via the `allow` list above.
exceptions = [
# CDDL-1.0 (copyleft) is pulled in only as a dev/profiling dependency via
# `inferno` -> `pprof` -> `lance-testing`; it is a test dependency that we
# do not distribute, so scope the allowance to `inferno` alone.
{ allow = ["CDDL-1.0"], crate = "inferno" },
]
exceptions = []
# Crates whose license cannot be determined from Cargo metadata but whose
# license we've manually confirmed from upstream. Keep this list minimal.
[[licenses.clarify]]
@@ -164,6 +167,11 @@ multiple-versions = "warn"
# Wildcard version requirements (`foo = "*"`) are a footgun — they let any
# future release in without review. Ban them outright.
wildcards = "deny"
# Lint every dependency declared by a workspace member against the shared
# `[workspace.dependencies]` table: any crate used by more than one member must
# go through `workspace = true`, and entries nothing uses are an error. This
# keeps versions from drifting between the core crate and the bindings.
workspace-dependencies = { duplicates = "deny", unused = "deny" }
# Internal workspace crates reference each other via `path = "..."`, which
# cargo-deny sees as a wildcard version. That's fine for private workspace
# members (not published to crates.io), so allow it specifically for paths.
docs/README.md
+11 -8
View File
@@ -47,22 +47,24 @@ pytest -vv python/tests/docs
### Checking typescript examples
The `@lancedb/lancedb` package must be built before running the tests:
The examples depend on `@lancedb/lancedb` at `file:../dist`, so the package must be
built before running the tests. This uses pnpm; see the
[Typescript contributing guide](../nodejs/CONTRIBUTING.md) for the toolchain setup.
```shell
pushd nodejs
npm ci
npm run build
pnpm install
pnpm build
popd
```
Then you can run the examples by going to the `nodejs/examples` directory and
running the tests like a normal npm package:
Then you can run the examples by going to the `nodejs/examples` directory, which is a
separate pnpm package with its own lockfile:
```shell
pushd nodejs/examples
npm ci
npm test
pnpm install
pnpm test
popd
```
@@ -84,6 +86,7 @@ The new files should be checked into the repository.
```shell
pushd nodejs
npm run docs
# `pnpm docs` would invoke pnpm's built-in `docs` command, not the script.
pnpm run docs
popd
```
docs/openapi.yml
+10 -1
View File
@@ -155,7 +155,7 @@ paths:
vector:
type: FixedSizeList
description: |
The targetted vector to search for. Required.
The targeted vector to search for. Required.
vector_column:
type: string
description: |
@@ -446,6 +446,15 @@ paths:
properties:
column:
type: string
name:
type: string
description: Optional name for the created index.
replace:
type: boolean
default: true
description: |
Whether to replace an existing index with the same resolved
name. Defaults to true.
metric_type:
type: string
nullable: false
docs/package-lock.json
Generated
-135
View File
@@ -1,135 +0,0 @@
{
"name": "lancedb-docs-test",
"version": "1.0.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "lancedb-docs-test",
"version": "1.0.0",
"license": "Apache 2",
"dependencies": {
"apache-arrow": "file:../node/node_modules/apache-arrow",
"vectordb": "file:../node"
},
"devDependencies": {
"@types/node": "^20.11.8",
"typescript": "^5.3.3"
}
},
"../node": {
"name": "vectordb",
"version": "0.21.2-beta.0",
"cpu": [
"x64",
"arm64"
],
"license": "Apache-2.0",
"os": [
"darwin",
"linux",
"win32"
],
"dependencies": {
"@neon-rs/load": "^0.0.74",
"axios": "^1.4.0"
},
"devDependencies": {
"@neon-rs/cli": "^0.0.160",
"@types/chai": "^4.3.4",
"@types/chai-as-promised": "^7.1.5",
"@types/mocha": "^10.0.1",
"@types/node": "^18.16.2",
"@types/sinon": "^10.0.15",
"@types/temp": "^0.9.1",
"@types/uuid": "^9.0.3",
"@typescript-eslint/eslint-plugin": "^5.59.1",
"apache-arrow-old": "npm:apache-arrow@13.0.0",
"cargo-cp-artifact": "^0.1",
"chai": "^4.3.7",
"chai-as-promised": "^7.1.1",
"eslint": "^8.39.0",
"eslint-config-standard-with-typescript": "^34.0.1",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-n": "^15.7.0",
"eslint-plugin-promise": "^6.1.1",
"mocha": "^10.2.0",
"openai": "^4.24.1",
"sinon": "^15.1.0",
"temp": "^0.9.4",
"ts-node": "^10.9.1",
"ts-node-dev": "^2.0.0",
"typedoc": "^0.24.7",
"typedoc-plugin-markdown": "^3.15.3",
"typescript": "^5.1.0",
"uuid": "^9.0.0"
},
"optionalDependencies": {
"@lancedb/vectordb-darwin-arm64": "0.21.2-beta.0",
"@lancedb/vectordb-darwin-x64": "0.21.2-beta.0",
"@lancedb/vectordb-linux-arm64-gnu": "0.21.2-beta.0",
"@lancedb/vectordb-linux-x64-gnu": "0.21.2-beta.0",
"@lancedb/vectordb-win32-x64-msvc": "0.21.2-beta.0"
},
"peerDependencies": {
"@apache-arrow/ts": "^14.0.2",
"apache-arrow": "^14.0.2"
}
},
"../node/node_modules/apache-arrow": {
"version": "14.0.2",
"license": "Apache-2.0",
"dependencies": {
"@types/command-line-args": "5.2.0",
"@types/command-line-usage": "5.0.2",
"@types/node": "20.3.0",
"@types/pad-left": "2.1.1",
"command-line-args": "5.2.1",
"command-line-usage": "7.0.1",
"flatbuffers": "23.5.26",
"json-bignum": "^0.0.3",
"pad-left": "^2.1.0",
"tslib": "^2.5.3"
},
"bin": {
"arrow2csv": "bin/arrow2csv.js"
}
},
"node_modules/@types/node": {
"version": "20.11.8",
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.8.tgz",
"integrity": "sha512-i7omyekpPTNdv4Jb/Rgqg0RU8YqLcNsI12quKSDkRXNfx7Wxdm6HhK1awT3xTgEkgxPn3bvnSpiEAc7a7Lpyow==",
"dev": true,
"dependencies": {
"undici-types": "~5.26.4"
}
},
"node_modules/apache-arrow": {
"resolved": "../node/node_modules/apache-arrow",
"link": true
},
"node_modules/typescript": {
"version": "5.3.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz",
"integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==",
"dev": true,
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
},
"engines": {
"node": ">=14.17"
}
},
"node_modules/undici-types": {
"version": "5.26.5",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
"integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
"dev": true
},
"node_modules/vectordb": {
"resolved": "../node",
"link": true
}
}
}
docs/package.json
-20
View File
@@ -1,20 +0,0 @@
{
"name": "lancedb-docs-test",
"version": "1.0.0",
"description": "auto-generated tests from doc",
"author": "dev@lancedb.com",
"license": "Apache 2",
"dependencies": {
"apache-arrow": "file:../node/node_modules/apache-arrow",
"vectordb": "file:../node"
},
"scripts": {
"build": "tsc -b && cd ../node && npm run build-release",
"example": "npm run build && node",
"test": "npm run build && ls dist/*.js | xargs -n 1 node"
},
"devDependencies": {
"@types/node": "^20.11.8",
"typescript": "^5.3.3"
}
}
docs/requirements.txt
+2 -2
View File
@@ -5,5 +5,5 @@ mkdocs-autorefs>=0.5,<=1.0
mkdocstrings[python]>=0.24,<1.0
griffe>=0.40,<1.0
mkdocs-render-swagger-plugin>=0.1.0
pydantic>=2.0,<3.0
mkdocs-redirects>=1.2.0
pydantic>=2.7.4,<3
mkdocs-redirects>=1.2.0
docs/src/java/java.md
+33 -1
View File
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
<dependency>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-core</artifactId>
<version>0.37.1-beta.0</version>
<version>0.40.0-beta.12</version>
</dependency>
```
@@ -55,6 +55,38 @@ LanceNamespace namespaceClient = LanceDbNamespaceClientBuilder.newBuilder()
| `region(String)` | AWS region (default: "us-east-1") | No |
| `config(String, String)` | Additional configuration parameters | No |
### Opening a Table with Vended Credentials
When the catalog vends temporary object store credentials, open the table through the
namespace client. The Lance dataset builder fetches the table location and storage options
from the catalog and refreshes the credentials when they expire.
```java
import com.lancedb.LanceDbNamespaceClientBuilder;
import org.lance.Dataset;
import org.lance.namespace.LanceNamespace;
import java.util.Arrays;
LanceNamespace namespaceClient = LanceDbNamespaceClientBuilder.newBuilder()
.apiKey(System.getenv("LANCEDB_API_KEY"))
.database(System.getenv("LANCEDB_DATABASE"))
// Set the endpoint for a LanceDB Enterprise deployment.
// .endpoint("https://your-enterprise-endpoint")
.build();
try (Dataset dataset = Dataset.open()
.namespaceClient(namespaceClient)
.tableId(Arrays.asList("my_namespace", "my_table"))
.build()) {
System.out.println("Rows: " + dataset.countRows());
}
```
Do not call `describeTable()` and then open the returned location with `Dataset.open(uri)`.
Opening through `namespaceClient()` is what applies the vended storage options and enables
automatic credential refresh. No object store credentials need to be passed by the application.
## Metadata Operations
### Creating a Namespace Path
docs/src/js/classes/AutoQuery.md
+518
View File
@@ -0,0 +1,518 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / AutoQuery
# Class: AutoQuery
A builder for automatic string searches.
Automatic search determines whether to use full-text or vector search from
the table revision selected for each execution. This builder exposes the
common operations supported by both query families.
## Extends
- `StandardQueryBase`&lt;`NativeQuery` \| `NativeVectorQuery`&gt;
## Properties
### inner
```ts
protected inner: Query | VectorQuery | Promise<Query | VectorQuery>;
```
#### Inherited from
`StandardQueryBase.inner`
## Methods
### analyzePlan()
```ts
analyzePlan(distributedMetrics?): Promise<string>
```
Executes the query and returns the physical query plan annotated with runtime metrics.
This is useful for debugging and performance analysis, as it shows how the query was executed
and includes metrics such as elapsed time, rows processed, and I/O statistics.
#### Parameters
* **distributedMetrics?**: [`AnalyzePlanDistributedMetrics`](../type-aliases/AnalyzePlanDistributedMetrics.md)
How distributed worker metrics are displayed for remote query plans.
Defaults to `"aggregate"`.
#### Returns
`Promise`&lt;`string`&gt;
A query execution plan with runtime metrics for each step.
#### Example
```ts
import * as lancedb from "@lancedb/lancedb"
const db = await lancedb.connect("./.lancedb");
const table = await db.createTable("my_table", [
{ vector: [1.1, 0.9], id: "1" },
]);
const plan = await table.query().nearestTo([0.5, 0.2]).analyzePlan();
Example output (with runtime metrics inlined):
AnalyzeExec verbose=true, metrics=[]
ProjectionExec: expr=[id@3 as id, vector@0 as vector, _distance@2 as _distance], metrics=[output_rows=1, elapsed_compute=3.292µs]
Take: columns="vector, _rowid, _distance, (id)", metrics=[output_rows=1, elapsed_compute=66.001µs, batches_processed=1, bytes_read=8, iops=1, requests=1]
CoalesceBatchesExec: target_batch_size=1024, metrics=[output_rows=1, elapsed_compute=3.333µs]
GlobalLimitExec: skip=0, fetch=10, metrics=[output_rows=1, elapsed_compute=167ns]
FilterExec: _distance@2 IS NOT NULL, metrics=[output_rows=1, elapsed_compute=8.542µs]
SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], metrics=[output_rows=1, elapsed_compute=63.25µs, row_replacements=1]
KNNVectorDistance: metric=l2, metrics=[output_rows=1, elapsed_compute=114.333µs, output_batches=1]
LanceScan: uri=/path/to/data, projection=[vector], row_id=true, row_addr=false, ordered=false, metrics=[output_rows=1, elapsed_compute=103.626µs, bytes_read=549, iops=2, requests=2]
```
#### Inherited from
`StandardQueryBase.analyzePlan`
***
### execute()
```ts
protected execute(options?): AsyncGenerator<RecordBatch<any>, void, unknown>
```
Execute the query and return the results as an
#### Parameters
* **options?**: `Partial`&lt;[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)&gt;
#### Returns
`AsyncGenerator`&lt;`RecordBatch`&lt;`any`&gt;, `void`, `unknown`&gt;
#### See
- AsyncIterator
of
- RecordBatch.
By default, LanceDb will use many threads to calculate results and, when
the result set is large, multiple batches will be processed at one time.
This readahead is limited however and backpressure will be applied if this
stream is consumed slowly (this constrains the maximum memory used by a
single query)
#### Inherited from
`StandardQueryBase.execute`
***
### explainPlan()
```ts
explainPlan(verbose): Promise<string>
```
Generates an explanation of the query execution plan.
#### Parameters
* **verbose**: `boolean` = `false`
If true, provides a more detailed explanation. Defaults to false.
#### Returns
`Promise`&lt;`string`&gt;
A Promise that resolves to a string containing the query execution plan explanation.
#### Example
```ts
import * as lancedb from "@lancedb/lancedb"
const db = await lancedb.connect("./.lancedb");
const table = await db.createTable("my_table", [
{ vector: [1.1, 0.9], id: "1" },
]);
const plan = await table.query().nearestTo([0.5, 0.2]).explainPlan();
```
#### Inherited from
`StandardQueryBase.explainPlan`
***
### fastSearch()
```ts
fastSearch(): this
```
Skip searching un-indexed data. This can make search faster, but will miss
any data that is not yet indexed.
Use [Table#optimize](Table.md#optimize) to index all un-indexed data.
#### Returns
`this`
#### Inherited from
`StandardQueryBase.fastSearch`
***
### ~~filter()~~
```ts
filter(predicate): this
```
A filter statement to be applied to this query.
#### Parameters
* **predicate**: `string`
#### Returns
`this`
#### See
where
#### Deprecated
Use `where` instead
#### Inherited from
`StandardQueryBase.filter`
***
### fullTextSearch()
```ts
fullTextSearch(query, options?): this
```
#### Parameters
* **query**: `string` \| [`FullTextQuery`](../interfaces/FullTextQuery.md)
* **options?**: `Partial`&lt;[`FullTextSearchOptions`](../interfaces/FullTextSearchOptions.md)&gt;
#### Returns
`this`
#### Inherited from
`StandardQueryBase.fullTextSearch`
***
### limit()
```ts
limit(limit): this
```
Set the maximum number of results to return.
By default, a plain search has no limit. If this method is not
called then every valid row from the table will be returned.
#### Parameters
* **limit**: `number`
#### Returns
`this`
#### Inherited from
`StandardQueryBase.limit`
***
### offset()
```ts
offset(offset): this
```
Set the number of rows to skip before returning results.
This is useful for pagination.
#### Parameters
* **offset**: `number`
#### Returns
`this`
#### Inherited from
`StandardQueryBase.offset`
***
### orderBy()
```ts
orderBy(ordering): this
```
Sort the results by the specified column(s).
#### Parameters
* **ordering**: [`ColumnOrdering`](../interfaces/ColumnOrdering.md) \| [`ColumnOrdering`](../interfaces/ColumnOrdering.md)[]
#### Returns
`this`
This query builder.
#### Inherited from
`StandardQueryBase.orderBy`
***
### outputSchema()
```ts
outputSchema(): Promise<Schema<any>>
```
Returns the schema of the output that will be returned by this query.
This can be used to inspect the types and names of the columns that will be
returned by the query before executing it.
#### Returns
`Promise`&lt;`Schema`&lt;`any`&gt;&gt;
An Arrow Schema describing the output columns.
#### Inherited from
`StandardQueryBase.outputSchema`
***
### select()
```ts
select(columns): this
```
Return only the specified columns.
By default a query will return all columns from the table. However, this can have
a very significant impact on latency. LanceDb stores data in a columnar fashion. This
means we can finely tune our I/O to select exactly the columns we need.
As a best practice you should always limit queries to the columns that you need. If you
pass in an array of column names then only those columns will be returned.
You can also use this method to create new "dynamic" columns based on your existing columns.
For example, you may not care about "a" or "b" but instead simply want "a + b". This is often
seen in the SELECT clause of an SQL query (e.g. `SELECT a+b FROM my_table`).
To create dynamic columns you can pass in a Map<string, string>. A column will be returned
for each entry in the map. The key provides the name of the column. The value is
an SQL string used to specify how the column is calculated.
For example, an SQL query might state `SELECT a + b AS combined, c`. The equivalent
input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
`this`
#### Example
```ts
new Map([["combined", "a + b"], ["c", "c"]])
Columns will always be returned in the order given, even if that order is different than
the order used when adding the data.
Note that you can pass in a `Record<string, string>` (e.g. an object literal). This method
uses `Object.entries` which should preserve the insertion order of the object. However,
object insertion order is easy to get wrong and `Map` is more foolproof.
```
#### Inherited from
`StandardQueryBase.select`
***
### toArray()
```ts
toArray(options?): Promise<any[]>
```
Collect the results as an array of objects.
#### Parameters
* **options?**: `Partial`&lt;[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)&gt;
#### Returns
`Promise`&lt;`any`[]&gt;
#### Inherited from
`StandardQueryBase.toArray`
***
### toArrow()
```ts
toArrow(options?): Promise<Table<any>>
```
Collect the results as an Arrow
#### Parameters
* **options?**: `Partial`&lt;[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)&gt;
#### Returns
`Promise`&lt;`Table`&lt;`any`&gt;&gt;
#### See
ArrowTable.
#### Inherited from
`StandardQueryBase.toArrow`
***
### useLsm()
```ts
useLsm(enable): this
```
Control MemWAL read routing for this query.
By default (unset), when the table carries a MemWAL write spec (see
[Table#setLsmWriteSpec](Table.md#setlsmwritespec)), reads are routed through the LSM scanner so
they also return data written via the `mergeInsert` LSM path that has not yet
been compacted into the base table (the active/frozen in-memory memtables and
the flushed generations), deduplicated by primary key; a table without a spec
reads the base table.
#### Parameters
* **enable**: `boolean`
`true` forces the LSM scanner and errors if the table has no
MemWAL write spec. `false` bypasses the MemWAL and reads the base table only,
even when a spec is present.
Note: the LSM scanner does not support every query shape (e.g. reranking,
hybrid search, `orderBy`). On a MemWAL table those shapes error unless
`useLsm(false)` is set, because a base-only read would silently exclude
un-compacted MemWAL data.
#### Returns
`this`
#### Inherited from
`StandardQueryBase.useLsm`
***
### where()
```ts
where(predicate): this
```
A filter statement to be applied to this query.
The filter should be supplied as an SQL query string. For example:
#### Parameters
* **predicate**: `string`
#### Returns
`this`
#### Example
```ts
x > 10
y > 0 AND y < 100
x > 5 OR y = 'test'
Filtering performance can often be improved by creating a scalar index
on the filter column(s).
Calling this multiple times combines the filters with a logical AND rather
than replacing the previous filter.
```
#### Inherited from
`StandardQueryBase.where`
***
### withRowId()
```ts
withRowId(): this
```
Whether to return the row id in the results.
This column can be used to match results between different queries. For
example, to match results from a full text search and a vector search in
order to perform hybrid search.
#### Returns
`this`
#### Inherited from
`StandardQueryBase.withRowId`
docs/src/js/classes/BlobFile.md
+158
View File
@@ -0,0 +1,158 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobFile
# Class: BlobFile
A lazy handle to blob bytes. Create one with [Table.fetchBlobFiles](Table.md#fetchblobfiles).
## Methods
### close()
```ts
close(): Promise<void>
```
Releases the handle. Reads after `close()` fail. Calling it again does
nothing.
#### Returns
`Promise`&lt;`void`&gt;
***
### isClosed()
```ts
isClosed(): boolean
```
Returns true after [BlobFile.close](BlobFile.md#close).
#### Returns
`boolean`
***
### read()
```ts
read(maxBytes?, options?): Promise<Buffer>
```
Reads from the cursor and advances the cursor.
Reads to the end when `maxBytes` is omitted, or at most `maxBytes` bytes
otherwise. Returns an empty buffer at the end of the blob.
[BlobFile.readRange](BlobFile.md#readrange) does not move the cursor.
#### Parameters
* **maxBytes?**: `bigint`
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;`Buffer`&gt;
***
### readRange()
```ts
readRange(
start,
end,
options?): Promise<Buffer>
```
Reads the half-open byte range `[start, end)`.
Fails when `end` is past the blob size. Does not move the cursor.
#### Parameters
* **start**: `bigint`
* **end**: `bigint`
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;`Buffer`&gt;
***
### readRanges()
```ts
readRanges(ranges, options?): Promise<Buffer[]>
```
Reads several half-open byte ranges. Returns one buffer per range, in
the order given.
Fails when any `end` is past the blob size. Does not move the cursor.
#### Parameters
* **ranges**: [`BlobRange`](../type-aliases/BlobRange.md)[]
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;`Buffer`[]&gt;
***
### seek()
```ts
seek(position): Promise<void>
```
Moves the cursor to `position`, in bytes from the start of the blob.
#### Parameters
* **position**: `bigint`
#### Returns
`Promise`&lt;`void`&gt;
***
### size()
```ts
size(): bigint
```
Returns the blob size in bytes.
#### Returns
`bigint`
***
### tell()
```ts
tell(): Promise<bigint>
```
Returns the cursor position in bytes.
#### Returns
`Promise`&lt;`bigint`&gt;
docs/src/js/classes/Branches.md
+25 -25
View File
@@ -37,6 +37,31 @@ latest and stays writable.
***
### cherryPick()
```ts
cherryPick(fromBranch, dryRun): Promise<CherryPickResult>
```
Cherry-pick a branch onto main.
Set `dryRun` to `true` to preview. A failed cherry-pick resolves
with `status: "failed"` instead of throwing.
#### Parameters
* **fromBranch**: `string`
Branch to cherry-pick from.
* **dryRun**: `boolean` = `false`
When true, only preview. Defaults to false.
#### Returns
`Promise`&lt;[`CherryPickResult`](../interfaces/CherryPickResult.md)&gt;
***
### create()
```ts
@@ -112,28 +137,3 @@ List all branches, mapping name to branch metadata.
#### Returns
`Promise`&lt;`Record`&lt;`string`, [`BranchContents`](BranchContents.md)&gt;&gt;
***
### merge()
```ts
merge(fromBranch, dryRun): Promise<MergeBranchResult>
```
Merge a branch into main.
Set `dryRun` to `true` to preview the merge. A rejected merge resolves
with `status: "rejected"` instead of throwing.
#### Parameters
* **fromBranch**: `string`
Branch to merge from.
* **dryRun**: `boolean` = `false`
When true, only preview the merge. Defaults to false.
#### Returns
`Promise`&lt;[`MergeBranchResult`](../interfaces/MergeBranchResult.md)&gt;
docs/src/js/classes/Catalog.md
+107
View File
@@ -0,0 +1,107 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / Catalog
# Class: Catalog
A remote catalog manages databases through the server's root namespace.
## Accessors
### uri
```ts
get uri(): string
```
The root namespace endpoint.
#### Returns
`string`
## Methods
### connectDatabase()
```ts
connectDatabase(name): Promise<Connection>
```
Connect to an existing database by its logical name.
#### Parameters
* **name**: `string`
#### Returns
`Promise`&lt;[`Connection`](Connection.md)&gt;
***
### createDatabase()
```ts
createDatabase(name, options): Promise<Connection>
```
Create a database, or open an existing database when existOk is true.
#### Parameters
* **name**: `string`
* **options** = `{}`
* **options.existOk?**: `boolean`
#### Returns
`Promise`&lt;[`Connection`](Connection.md)&gt;
***
### dropDatabase()
```ts
dropDatabase(name, options): Promise<void>
```
Drop an empty database. The server rejects nonempty databases.
#### Parameters
* **name**: `string`
* **options** = `{}`
* **options.ignoreMissing?**: `boolean`
#### Returns
`Promise`&lt;`void`&gt;
***
### listDatabases()
```ts
listDatabases(options): Promise<ListDatabasesResponse>
```
List one page of databases; pass pageToken from a response for the next page.
#### Parameters
* **options** = `{}`
* **options.limit?**: `number`
* **options.pageToken?**: `string`
#### Returns
`Promise`&lt;[`ListDatabasesResponse`](../interfaces/ListDatabasesResponse.md)&gt;
docs/src/js/classes/Connection.md
+395 -54
View File
@@ -169,6 +169,47 @@ Creates a new empty Table
***
### createMaterializedView()
```ts
abstract createMaterializedView(
name,
source,
options?): Promise<MaterializedView>
```
Define a materialized view named `name` over the table `source`.
The view is populated before creation returns. Set `withNoData` to create
only its definition and empty backing table. The view is a normal table:
it can be queried, indexed and searched, and it appears in `tableNames`.
The source table must have stable row ids (create it with
the `newTableEnableStableRowIds` storage option); they keep the view's
provenance valid across source compactions and cannot be enabled after
a table exists.
#### Parameters
* **name**: `string`
* **source**: `string`
* **options?**
* **options.limit?**: `number`
* **options.select?**: [`MaterializedViewSelect`](../type-aliases/MaterializedViewSelect.md)
* **options.where?**: `string`
* **options.withNoData?**: `boolean`
#### Returns
`Promise`&lt;[`MaterializedView`](MaterializedView.md)&gt;
***
### createNamespace()
```ts
@@ -278,6 +319,38 @@ Creates a new Table and initialize it with new data.
***
### createView()
```ts
abstract createView(
name,
query,
namespacePath?): Promise<ViewDescription>
```
Create a view: a named query the database plans on every read.
The query is planned once, at creation, so one that cannot be planned is
rejected now rather than at the first read. A view holds no rows, and its
readers see its sources as they are at read time.
There is no replace: a name already taken is an error, and changing a
view is a drop followed by a create.
#### Parameters
* **name**: `string`
* **query**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;[`ViewDescription`](../interfaces/ViewDescription.md)&gt;
***
### describeNamespace()
```ts
@@ -301,6 +374,27 @@ The namespace's properties
***
### describeView()
```ts
abstract describeView(name, namespacePath?): Promise<ViewDescription>
```
What this database records about the view named `name`: its defining
query and the schema that query resolved to.
#### Parameters
* **name**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;[`ViewDescription`](../interfaces/ViewDescription.md)&gt;
***
### display()
```ts
@@ -334,6 +428,54 @@ Drop all tables in the database.
***
### dropMaterializedView()
```ts
abstract dropMaterializedView(name, namespacePath?): Promise<void>
```
Drop the materialized view named `name`.
The view may become unavailable before physical cleanup finishes. Use
[dropMaterializedViewAsync](Connection.md#dropmaterializedviewasync) to retain and wait for the cleanup job.
Rejects a table that exists but is not a materialized view.
#### Parameters
* **name**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;`void`&gt;
***
### dropMaterializedViewAsync()
```ts
abstract dropMaterializedViewAsync(name, namespacePath?): Promise<Job>
```
Start dropping the materialized view named `name` and return its cleanup
job without waiting for completion.
Rejects a table that exists but is not a materialized view.
#### Parameters
* **name**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;[`Job`](Job.md)&gt;
***
### dropNamespace()
```ts
@@ -386,23 +528,73 @@ Drop an existing table.
***
### getJob()
### dropTableAsync()
```ts
abstract getJob(jobId): Promise<null | JobDescription>
abstract dropTableAsync(name, namespacePath?): Promise<Job>
```
Describe a single server-side job by id.
Start dropping a table and return its cleanup job.
Resolves to `null` when the server has no such job.
The table may become unavailable before its data files are removed. Wait
on the returned job to know when cleanup has finished.
#### Parameters
* **jobId**: `string`
* **name**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;`null` \| [`JobDescription`](../interfaces/JobDescription.md)&gt;
`Promise`&lt;[`Job`](Job.md)&gt;
***
### dropView()
```ts
abstract dropView(name, namespacePath?): Promise<void>
```
Drop the view named `name` and wait for its definition to be deleted.
The tables it reads are untouched: a view holds no rows of its own. Use
[dropViewAsync](Connection.md#dropviewasync) to retain the cleanup job instead of waiting on it.
#### Parameters
* **name**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;`void`&gt;
***
### dropViewAsync()
```ts
abstract dropViewAsync(name, namespacePath?): Promise<Job>
```
Start dropping the view named `name` and return the job deleting its
definition, without waiting for completion.
The name is free before this resolves. When nothing was bound to it, the
returned job is already finished and has no id.
#### Parameters
* **name**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;[`Job`](Job.md)&gt;
***
@@ -420,48 +612,6 @@ Return true if the connection has not been closed
***
### job()
```ts
abstract job(jobId): Job
```
A [Job](Job.md) handle for a server-side job by id.
The handle is constructed without a server round trip; an unknown id
surfaces when the handle is used. Dropping the handle has no effect on
the job itself.
#### Parameters
* **jobId**: `string`
#### Returns
[`Job`](Job.md)
***
### jobHistory()
```ts
abstract jobHistory(jobId?): Promise<Table<any>>
```
The lifecycle event history of a server-side job, as an Arrow table.
Lists history across all jobs when `jobId` is omitted.
#### Parameters
* **jobId?**: `string`
#### Returns
`Promise`&lt;`Table`&lt;`any`&gt;&gt;
***
### listJobs()
```ts
@@ -476,6 +626,22 @@ List server-side jobs across the database's tables.
***
### listMaterializedViews()
```ts
abstract listMaterializedViews(): Promise<string[]>
```
The names of the materialized views in this database.
Found by reading every table's schema, so this costs an open per table.
#### Returns
`Promise`&lt;`string`[]&gt;
***
### listNamespaces()
```ts
@@ -506,6 +672,134 @@ Child namespace names and
***
### listTables()
#### listTables(options)
```ts
abstract listTables(options?): Promise<ListTablesResponse>
```
List a page of the tables in this database.
To retrieve the tables after the page, pass the `pageToken` the response
carries back in. A page can be shorter than `limit` without being the last
one, so walk until a response carries no page token:
```ts
const names = [];
let pageToken = undefined;
do {
const page = await conn.listTables({ pageToken, limit: 100 });
names.push(...page.tables);
pageToken = page.pageToken;
} while (pageToken);
```
##### Parameters
* **options?**: `Partial`&lt;[`ListTablesOptions`](../interfaces/ListTablesOptions.md)&gt;
Pagination options
(`pageToken`, `limit`).
##### Returns
`Promise`&lt;[`ListTablesResponse`](../interfaces/ListTablesResponse.md)&gt;
A page of table names and an
optional token for the tables after it.
#### listTables(namespacePath, options)
```ts
abstract listTables(namespacePath?, options?): Promise<ListTablesResponse>
```
List a page of the tables in this database.
##### Parameters
* **namespacePath?**: `string`[]
The namespace path to list tables from
(defaults to root namespace)
* **options?**: `Partial`&lt;[`ListTablesOptions`](../interfaces/ListTablesOptions.md)&gt;
Pagination options
(`pageToken`, `limit`).
##### Returns
`Promise`&lt;[`ListTablesResponse`](../interfaces/ListTablesResponse.md)&gt;
A page of table names and an
optional token for the tables after it.
***
### listViews()
```ts
abstract listViews(namespacePath?): Promise<string[]>
```
The names of the views in one namespace.
Names only; a definition comes from [describeView](Connection.md#describeview).
#### Parameters
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;`string`[]&gt;
***
### openJob()
```ts
abstract openJob(jobId): Promise<Job>
```
Open a server-side job by id, returning a handle with its record already
populated. Rejects when the server has no such job, the way
[Connection.openTable](Connection.md#opentable) does for a missing table.
The returned [Job](Job.md) answers for its own state, specification,
result, failure and event history, so there is no separate
connection-level call for any of them.
#### Parameters
* **jobId**: `string`
#### Returns
`Promise`&lt;[`Job`](Job.md)&gt;
***
### openMaterializedView()
```ts
abstract openMaterializedView(name): Promise<MaterializedView>
```
Open the materialized view named `name`.
Rejects a table that exists but is not a materialized view.
#### Parameters
* **name**: `string`
#### Returns
`Promise`&lt;[`MaterializedView`](MaterializedView.md)&gt;
***
### openTable()
```ts
@@ -515,18 +809,13 @@ abstract openTable(
options?): Promise<Table>
```
Open a table in the database.
#### Parameters
* **name**: `string`
The name of the table
* **namespacePath?**: `string`[]
The namespace path of the table (defaults to root namespace)
* **options?**: `Partial`&lt;[`OpenTableOptions`](../interfaces/OpenTableOptions.md)&gt;
Additional options
#### Returns
@@ -534,6 +823,28 @@ Open a table in the database.
***
### pauseJob()
```ts
abstract pauseJob(jobId): Promise<string>
```
Pause a server-side job by id.
The job's workers drain and it stays parked until resumed. Resolves to
"pausing", "already_paused", or "committing" -- a job finalizing its
results cannot be parked; retry shortly.
#### Parameters
* **jobId**: `string`
#### Returns
`Promise`&lt;`string`&gt;
***
### renameTable()
```ts
@@ -567,7 +878,29 @@ a "not supported" error.
***
### tableNames()
### resumeJob()
```ts
abstract resumeJob(jobId): Promise<string>
```
Resume a paused server-side job by id.
Its workers pick their work back up from checkpoints. Resolves to
"resumed", "still_pausing" -- the pause's worker drain is not confirmed
yet; retry shortly -- or "not_paused".
#### Parameters
* **jobId**: `string`
#### Returns
`Promise`&lt;`string`&gt;
***
### ~~tableNames()~~
#### tableNames(options)
@@ -589,6 +922,10 @@ Tables will be returned in lexicographical order.
`Promise`&lt;`string`[]&gt;
##### Deprecated
Use [Connection.listTables](Connection.md#listtables) instead.
#### tableNames(namespacePath, options)
```ts
@@ -611,3 +948,7 @@ Tables will be returned in lexicographical order.
##### Returns
`Promise`&lt;`string`[]&gt;
##### Deprecated
Use [Connection.listTables](Connection.md#listtables) instead.
docs/src/js/classes/Job.md
+159 -12
View File
@@ -8,19 +8,46 @@
A handle to an operation that may still be running.
## Constructors
The operation may already be complete when the handle is created.
### new Job()
The detail getters read what the handle last observed. Submitting an
operation returns only a job id, so populating them eagerly would cost an
extra round trip on every call:
- [Job.refresh](Job.md#refresh) and [Job.status](Job.md#status) fetch the whole record.
- [Job.wait](Job.md#wait) records the terminal state it establishes, but not the
rest of the record.
- Everything is null until one of those runs.
## Accessors
### creationMs
```ts
new Job(): Job
get creationMs(): null | number
```
When the job was created, in milliseconds since the epoch.
#### Returns
[`Job`](Job.md)
`null` \| `number`
## Accessors
***
### failure
```ts
get failure(): null | JobFailureInfo
```
Why the job failed, when it failed and the server reports a reason.
#### Returns
`null` \| [`JobFailureInfo`](../interfaces/JobFailureInfo.md)
***
### id
@@ -28,8 +55,69 @@ new Job(): Job
get id(): null | string
```
Identifies the operation on the server that is running it. Operations
that run in this process have no server id. The value is opaque.
Identifies the operation on the server that is running it.
Operations that run in this process have no server id. The value is
opaque: parsing it or storing it to resume the job later is not supported.
#### Returns
`null` \| `string`
***
### jobType
```ts
get jobType(): null | string
```
The job's type, as the server names it. Null for an in-process job, which
has no server-side record.
#### Returns
`null` \| `string`
***
### result
```ts
get result(): any
```
The job-type-specific terminal result. Null until the job succeeds, so a
job that never terminates reports its progress through [Job.events](Job.md#events)
instead.
#### Returns
`any`
***
### spec
```ts
get spec(): any
```
The job-type-specific specification it was submitted with.
#### Returns
`any`
***
### state
```ts
get state(): null | string
```
The last observed lifecycle state, without contacting the backend.
#### Returns
@@ -51,18 +139,61 @@ Request cancellation. Cancelling a finished operation is a no-op.
***
### events()
```ts
events(options?): Promise<Table<any>>
```
This job's recorded lifecycle events.
Where the getters above report a terminal result only once the job reaches
one, events are written as the job runs and outlive the workers that
produced them. A distributed job records a `claim`/`claim_complete` pair
per unit of work, each carrying `rows_processed`, so a job that never
finishes still accounts for what it did.
The server caps results at 1000 rows by default and 10,000 at most, and
truncates without saying so, so pass `limit` for a job that emits an event
per fragment. `filter` is a SQL-like expression over the `state`,
`updated_by`, `emitted_from`, `emitted_by`, and `claim_entity` columns.
#### Parameters
* **options?**: [`JobEventsOptions`](../interfaces/JobEventsOptions.md)
#### Returns
`Promise`&lt;`Table`&lt;`any`&gt;&gt;
***
### refresh()
```ts
refresh(): Promise<void>
```
Ask the backend for this job's current state, and for a server-side job
its full record, then cache it for the getters above.
#### Returns
`Promise`&lt;`void`&gt;
***
### status()
```ts
status(): Promise<string>
```
The operation's current lifecycle state: "running", "finished",
"failed", or "cancelled".
The operation's current lifecycle state: "running", "finished", "failed",
or "cancelled".
A point snapshot; unlike [Job.wait](Job.md#wait) it does not block or reject
on a terminal failure state. States a newer server reports that this
client version does not know pass through as-is.
A point snapshot; unlike [Job.wait](Job.md#wait) it does not block or reject on a
terminal failure state. Also refreshes the getters above.
#### Returns
@@ -70,6 +201,22 @@ client version does not know pass through as-is.
***
### toString()
```ts
toString(): string
```
Every field the handle currently knows, one per line, with the JSON
payloads indented -- a refresh job's spec and result are the point of
printing it.
#### Returns
`string`
***
### wait()
```ts
docs/src/js/classes/MaterializedView.md
+101
View File
@@ -0,0 +1,101 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / MaterializedView
# Class: MaterializedView
A handle on a materialized view: its table plus its definition.
Obtained from [Connection#createMaterializedView](Connection.md#creatematerializedview) or
[Connection#openMaterializedView](Connection.md#openmaterializedview). The view is a normal table --
queries, indexes and search all apply through [MaterializedView#table](MaterializedView.md#table)
-- whose contents are maintained by [MaterializedView#refresh](MaterializedView.md#refresh).
## Constructors
### new MaterializedView()
```ts
new MaterializedView(table): MaterializedView
```
#### Parameters
* **table**: [`Table`](Table.md)
#### Returns
[`MaterializedView`](MaterializedView.md)
## Accessors
### name
```ts
get name(): string
```
#### Returns
`string`
## Methods
### definition()
```ts
definition(): Promise<MaterializedViewDefinition>
```
The query that defines the view.
#### Returns
`Promise`&lt;[`MaterializedViewDefinition`](../interfaces/MaterializedViewDefinition.md)&gt;
***
### refresh()
```ts
refresh(options?): Promise<RefreshMaterializedViewResult>
```
Recompute the view from its source.
The refresh is incremental when the source's changes can be reconciled
into the view -- rows added, changed or removed since the last one --
and otherwise rebuilds. `full` forces a rebuild; `sourceVersion`
refreshes to that source version instead of the latest.
Concurrent refreshes of one view do not duplicate its rows. Two that
plan the same source rows conflict on commit, and the loser throws
rather than writing them a second time.
#### Parameters
* **options?**
* **options.full?**: `boolean`
* **options.sourceVersion?**: `number`
#### Returns
`Promise`&lt;[`RefreshMaterializedViewResult`](../interfaces/RefreshMaterializedViewResult.md)&gt;
***
### table()
```ts
table(): Table
```
The view, as the table it is.
#### Returns
[`Table`](Table.md)
docs/src/js/classes/MergeInsertBuilder.md
+1 -1
View File
@@ -141,7 +141,7 @@ Currently this causes multiple copies of the row to be created
but that behavior is subject to change.
An optional condition may be specified. If it is, then only
matched rows that satisfy the condtion will be updated. Any
matched rows that satisfy the condition will be updated. Any
rows that do not satisfy the condition will be left as they
are. Failing to satisfy the condition does not cause a
"matched row" to become a "not matched" row.
docs/src/js/classes/OAuthSession.md
+105
View File
@@ -0,0 +1,105 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / OAuthSession
# Class: OAuthSession
Explicit OAuth session lifecycle for the persistent token cache: eager
`login`, non-secret `status`, and local `logout`.
A session is built from the same [OAuthConfig](../interfaces/OAuthConfig.md) used to connect
(including its `tokenCache` options). A connection created with the same
configuration shares the cache, so logging in here prepares tokens for
later processes without any database request.
`login` always runs the configured interactive flow and replaces the cached
session (the most recent login wins). `logout` removes only the local
credential; it does not revoke anything with the provider and does not sign
out of a browser SSO session.
## Example
```typescript
const config: OAuthConfig = {
issuerUrl: "https://issuer.example.com",
clientId: "my-app",
scopes: ["openid", "offline_access"],
flow: OAuthFlowType.DeviceCode,
tokenCache: { cacheDir: "/tmp/my-app/oauth-cache" },
};
const session = new OAuthSession(config);
const status = await session.login();
```
## Constructors
### new OAuthSession()
```ts
new OAuthSession(config): OAuthSession
```
Create a session manager for the given OAuth configuration.
#### Parameters
* **config**: [`OAuthConfig`](../interfaces/OAuthConfig.md)
#### Returns
[`OAuthSession`](OAuthSession.md)
## Methods
### login()
```ts
login(): Promise<SessionStatus>
```
Eagerly run the configured authentication flow and store the session.
A successful login always replaces any prior cached session for this
identity; if the provider does not issue a refresh token (for example
without `offline_access`), the previous record is removed and the status
reports `refreshable == false`.
#### Returns
`Promise`&lt;[`SessionStatus`](../interfaces/SessionStatus.md)&gt;
***
### logout()
```ts
logout(): Promise<SessionLogout>
```
Remove the matching local cached credential.
This only deletes the local cache entry. It does not revoke the refresh
token with the provider and does not sign out of a browser SSO session.
Repeated calls succeed; `removed` reports whether a credential existed.
#### Returns
`Promise`&lt;[`SessionLogout`](../interfaces/SessionLogout.md)&gt;
***
### status()
```ts
status(): Promise<SessionStatus>
```
Report whether a matching cached session exists, with safe metadata.
This never contacts the identity provider and never exposes token values.
#### Returns
`Promise`&lt;[`SessionStatus`](../interfaces/SessionStatus.md)&gt;
docs/src/js/classes/Query.md
+1 -1
View File
@@ -424,7 +424,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
docs/src/js/classes/QueryBase.md
+1 -1
View File
@@ -193,7 +193,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
docs/src/js/classes/Table.md
+326 -8
View File
@@ -69,14 +69,34 @@ abstract addColumns(newColumnTransforms): Promise<AddColumnsResult>
Add new columns with defined values.
The `{ computed }` form stores the expression rather than evaluating it
now: the column is committed with no values, and rows get them from
[Table#refreshColumn](Table.md#refreshcolumn). Declaring one therefore costs the same on a
large table as on an empty one.
A refresh also recomputes the rows whose inputs changed since they were
computed, so a mutated input is reflected by the next refresh. While a
declaration reads a column, that column cannot be renamed, retyped or
dropped.
On LanceDB Cloud and Enterprise the expression is planned by the
server, and the refresh runs as a server job -- see
[Table#refreshColumnAsync](Table.md#refreshcolumnasync).
#### Parameters
* **newColumnTransforms**: `Field`&lt;`any`&gt; \| `Field`&lt;`any`&gt;[] \| `Schema`&lt;`any`&gt; \| [`AddColumnsSql`](../interfaces/AddColumnsSql.md)[]
* **newColumnTransforms**:
\| `Field`&lt;`any`&gt;
\| `Field`&lt;`any`&gt;[]
\| `Schema`&lt;`any`&gt;
\| [`AddColumnsSql`](../interfaces/AddColumnsSql.md)[]
\| `object`
Either:
- An array of objects with column names and SQL expressions to calculate values
- A single Arrow Field defining one column with its data type (column will be initialized with null values)
- An array of Arrow Fields defining columns with their data types (columns will be initialized with null values)
- An Arrow Schema defining columns with their data types (columns will be initialized with null values)
- `{ computed }`, declaring columns defined by a SQL expression whose type and inputs are derived from it
#### Returns
@@ -85,6 +105,13 @@ Add new columns with defined values.
A promise that resolves to an object
containing the new version number of the table after adding the columns.
#### Example
```ts
await table.addColumns({ computed: [{ name: "doubled", valueSql: "x * 2" }] });
const { rowsFilled } = await table.refreshColumn("doubled");
```
***
### alterColumns()
@@ -110,6 +137,20 @@ containing the new version number of the table after altering the columns.
***
### blobColumns()
```ts
abstract blobColumns(): Promise<string[]>
```
Blob v2 columns, including nested dotted paths.
#### Returns
`Promise`&lt;`string`[]&gt;
***
### branches()
```ts
@@ -186,6 +227,39 @@ version of the table.
***
### checkpointLsm()
```ts
abstract checkpointLsm(): Promise<void>
```
Converge this table's LSM write path into its base table.
Seals once, then triggers compaction and polls until the L0 that existed
at the start is gone. The target set is fixed at the start, so
generations created *during* the checkpoint are ignored — that is what
lets it terminate under write load, and what makes it best-effort: it
converges the fresh tier as of some instant. Idempotent, abandonable at
any point, and safe to run on a cadence.
There is no liveness bound — the compactor pool is shared across tables,
so a checkpoint queued behind unrelated work looks exactly like one that
is merging. The caller owns the deadline.
#### Returns
`Promise`&lt;`void`&gt;
#### Example
```ts
const before = await table.getLsmStats();
await table.checkpointLsm();
const after = await table.getLsmStats();
```
***
### close()
```ts
@@ -223,6 +297,24 @@ It is a no-op when no writers are cached.
***
### compactLsm()
```ts
abstract compactLsm(): Promise<void>
```
Trigger a background L0 → base compaction pass per bucket.
Returns once the passes are *dispatched*, not once they finish — watch
[Table#getLsmStats](Table.md#getlsmstats) for progress, or use
[Table#checkpointLsm](Table.md#checkpointlsm) to wait for convergence.
#### Returns
`Promise`&lt;`void`&gt;
***
### countRows()
```ts
@@ -421,6 +513,145 @@ Drop an index from the table.
***
### fetchBlobFiles()
```ts
abstract fetchBlobFiles(
column,
rowIds,
options?): Promise<(null | BlobFile)[]>
```
Opens lazy blob handles for `column` at the given row IDs using the
table's current checkout.
Preserves input order, duplicates, and nulls. Use this for large payloads.
See [Table.fetchBlobs](Table.md#fetchblobs) for row-ID validity across versions.
#### Parameters
* **column**: `string`
* **rowIds**: readonly (`number` \| `bigint`)[]
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;(`null` \| [`BlobFile`](BlobFile.md))[]&gt;
***
### fetchBlobs()
```ts
abstract fetchBlobs(
column,
rowIds,
options?): Promise<(null | Buffer)[]>
```
Bytes for `column` at row IDs from [Query.withRowId](Query.md#withrowid).
Reads the table's current checkout. IDs from another version can fail after
compaction unless stable row ids are enabled. Results keep input order and
duplicates. Null blobs are `null`. Empty blobs are empty buffers.
Remote servers limit each request to 1024 row IDs and 64 MiB of blob bytes.
The client splits requests automatically and reads an individual larger
blob through the Range route. This method still materializes all bytes in
memory; use [Table.fetchBlobFiles](Table.md#fetchblobfiles) for large values.
#### Parameters
* **column**: `string`
* **rowIds**: readonly (`number` \| `bigint`)[]
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;(`null` \| `Buffer`)[]&gt;
***
### flushLsm()
```ts
abstract flushLsm(): Promise<void>
```
Seal every bucket's active memtable into a new L0 generation.
Returns once the seal is committed. Sealing an empty memtable is a no-op,
so this is safe to call repeatedly.
#### Returns
`Promise`&lt;`void`&gt;
***
### functionErrors()
```ts
abstract functionErrors(options?): Promise<FunctionErrors>
```
The per-row errors Function refreshes recorded on this table.
A refresh running under a skip policy records each row it skipped with
the input that failed and the error. This lists those records, newest
job first, plus a summary for any fragment whose per-row detail was
capped. LanceDB Cloud and Enterprise only; reading errors needs read
access to the table, since a message carries the value that failed.
#### Parameters
* **options?**: [`FunctionErrorsOptions`](../interfaces/FunctionErrorsOptions.md)
Optional filters: `jobId`,
`column`, and `limit` (server default 10000, cap 100000).
#### Returns
`Promise`&lt;[`FunctionErrors`](../interfaces/FunctionErrors.md)&gt;
The records, the capped fragments,
and whether the listing stopped at its limit.
#### Example
```ts
const { records, truncated } = await table.functionErrors({ column: "embedding" });
```
***
### getLsmStats()
```ts
abstract getLsmStats(includeGenerationRows?): Promise<undefined | LsmStats>
```
Read live per-bucket LSM state.
Answers "how far behind is my fresh tier", "which bucket is hot", and
"why is my fresh-tier vector search brute-force". Mutates no table state.
Resolves to `undefined` only when the LSM write path is not enabled.
#### Parameters
* **includeGenerationRows?**: `boolean`
Also count rows per L0 generation.
Off by default because each count opens an uncached Lance dataset.
#### Returns
`Promise`&lt;`undefined` \| [`LsmStats`](../interfaces/LsmStats.md)&gt;
***
### getLsmWriteSpec()
```ts
@@ -431,9 +662,10 @@ Read the [LsmWriteSpec](../interfaces/LsmWriteSpec.md) currently installed on th
Resolves to `undefined` when the MemWAL LSM write path is not enabled (no
spec has been set, or it was removed with [Table#unsetLsmWriteSpec](Table.md#unsetlsmwritespec)).
The returned spec — including its `maintainedIndexes` and
`writerConfigDefaults` — mirrors what was passed to
[Table#setLsmWriteSpec](Table.md#setlsmwritespec).
The returned spec mirrors what was passed to
[Table#setLsmWriteSpec](Table.md#setlsmwritespec), except that `maintainedIndexes` always
reports the concrete list resolved when the spec was set — `undefined`
never round-trips.
#### Returns
@@ -555,9 +787,17 @@ List all the versions of the table
abstract mergeInsert(on): MergeInsertBuilder
```
Create a [MergeInsertBuilder](MergeInsertBuilder.md), which combines new data with the
existing table in a single transaction — inserting, updating and deleting
rows depending on how they match.
#### Parameters
* **on**: `string` \| `string`[]
The column, or columns, to match source rows against target
rows on. Typically a key or id column. Several columns match on the
composite key: a source row updates a target row only when it agrees on
every one of them.
#### Returns
@@ -717,6 +957,67 @@ for await (const batch of table.query()) {
***
### refreshColumn()
```ts
abstract refreshColumn(column): Promise<RefreshColumnResult>
```
Fill the rows of a computed column that hold no value yet.
Rows appended since the last refresh are filled by the next one, and
rows whose inputs changed since they were computed are recomputed;
everything else is left as it is. Local tables only: a remote refresh
runs as a server job, through [Table#refreshColumnAsync](Table.md#refreshcolumnasync).
#### Parameters
* **column**: `string`
The name of the computed column to fill.
#### Returns
`Promise`&lt;[`RefreshColumnResult`](../interfaces/RefreshColumnResult.md)&gt;
A promise that resolves to the
number of rows filled and the new version number of the table.
***
### refreshColumnAsync()
```ts
abstract refreshColumnAsync(column): Promise<Job>
```
Like [Table#refreshColumn](Table.md#refreshcolumn), but returns a handle to the refresh
job instead of blocking until it completes.
The job may already be complete when returned; callers must not assume
the column is filled until [Job.wait](Job.md#wait) resolves. Invalid input --
an unknown column, or one that is not computed -- rejects here rather
than failing the job. On local tables the job runs in-process; on
LanceDB Cloud and Enterprise it is the server's backfill job.
#### Parameters
* **column**: `string`
The name of the computed column to fill.
#### Returns
`Promise`&lt;[`Job`](Job.md)&gt;
#### Example
```ts
const job = await table.refreshColumnAsync("doubled");
await job.wait();
console.log(await job.status()); // "finished"
```
***
### restore()
```ts
@@ -760,7 +1061,7 @@ Get the schema of the table.
abstract search(
query,
queryType?,
ftsColumns?): Query | VectorQuery
ftsColumns?): Query | VectorQuery | AutoQuery
```
Create a search query to find the nearest neighbors
@@ -782,7 +1083,7 @@ of the given query
#### Returns
[`Query`](Query.md) \| [`VectorQuery`](VectorQuery.md)
[`Query`](Query.md) \| [`VectorQuery`](VectorQuery.md) \| [`AutoQuery`](AutoQuery.md)
***
@@ -806,6 +1107,11 @@ All variants require the table to have an unenforced primary key
([Table#setUnenforcedPrimaryKey](Table.md#setunenforcedprimarykey)); bucket sharding additionally
requires it to be the single column being bucketed.
Omitting `maintainedIndexes` maintains every index on the table, resolved
here, failing if one cannot be maintained — name them to install anyway.
Naming them pins an exact set, and a still-building index is rejected
rather than quietly omitted.
#### Parameters
* **spec**: [`LsmWriteSpec`](../interfaces/LsmWriteSpec.md)
@@ -1071,11 +1377,11 @@ value is 0")
Note: if your condition is something like "some_id_column == 7" and
you are updating many rows (with different ids) then you will get
better performance with a single [`merge_insert`] call instead of
repeatedly calilng this method.
repeatedly calling this method.
##### Parameters
* **updates**: `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **updates**: `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
the
columns to update
@@ -1105,6 +1411,18 @@ abstract updateFieldMetadata(updates): Promise<UpdateFieldMetadataResult>
Update per-field (column) metadata.
The following keys are treated specially, by convention, and should be
used when appropriate:
- `lancedb:description`: for a human-readable description of a field.
- `lancedb:tag:<name>`: for a user-defined key-value tag, where the suffix
names the tag category; e.g. `lancedb:tag:model: "clip"`.
- `lancedb:logical-column`: for a column grouping; e.g. `feature_v1` and
`feature_v2` might be in the same logical column.
- `lancedb:status`: for status options (`production`, `candidate`,
`deprecated`, `archived`) to designate the current life cycle state of
this column.
#### Parameters
* **updates**: [`FieldMetadataUpdate`](../interfaces/FieldMetadataUpdate.md)[]
docs/src/js/classes/TakeQuery.md
+1 -1
View File
@@ -200,7 +200,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
docs/src/js/classes/VectorQuery.md
+1 -1
View File
@@ -673,7 +673,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
docs/src/js/enumerations/ClientAuthMethod.md
+48
View File
@@ -0,0 +1,48 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / ClientAuthMethod
# Enumeration: ClientAuthMethod
How the client authenticates to the OAuth token endpoint.
The method applies to every OAuth request that carries client
authentication: client-credentials, authorization-code exchange,
refresh-token, and device-authorization requests. The Azure managed
identity flow ignores this option.
## Enumeration Members
### ClientSecretBasic
```ts
ClientSecretBasic: "client_secret_basic";
```
HTTP Basic authentication. This is the RFC 6749 recommended method and
the normal default for confidential clients, including default Okta
applications. Requires `clientSecret`.
***
### ClientSecretPost
```ts
ClientSecretPost: "client_secret_post";
```
Credentials in the request body, for providers configured to require it.
Requires `clientSecret`.
***
### None
```ts
None: "none";
```
No client authentication, for public clients using PKCE or the device
flow. Cannot be combined with `clientSecret`.
docs/src/js/enumerations/OAuthFlowType.md
+20
View File
@@ -10,6 +10,16 @@ OAuth authentication flow types.
## Enumeration Members
### AuthorizationCode
```ts
AuthorizationCode: "authorization_code";
```
Interactive Authorization Code grant, using PKCE by default.
***
### AzureManagedIdentity
```ts
@@ -27,3 +37,13 @@ ClientCredentials: "client_credentials";
```
Client Credentials grant (service-to-service / M2M).
***
### DeviceCode
```ts
DeviceCode: "device_code";
```
Device Authorization grant for CLI and headless environments.
docs/src/js/functions/blob.md
+55
View File
@@ -0,0 +1,55 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / blob
# Function: blob()
```ts
function blob(name, options): Field
```
Declares a `lance.blob.v2` column.
Query results are descriptors, not payload bytes. Use [Table.fetchBlobs](../classes/Table.md#fetchblobs)
or [Table.fetchBlobFiles](../classes/Table.md#fetchblobfiles) to read bytes.
## Parameters
* **name**: `string`
* **options**: [`BlobOptions`](../type-aliases/BlobOptions.md) = `{}`
## Returns
`Field`
## Example
```ts
import { readFile } from "node:fs/promises";
import { Field, Int64, Schema } from "apache-arrow";
import { blob, connect } from "@lancedb/lancedb";
const db = await connect("./data");
const video = await readFile("clip.mp4");
const table = await db.createTable(
"videos",
[{ id: 1n, video }],
{
schema: new Schema([
new Field("id", new Int64()),
blob("video"),
]),
},
);
const rows = await table.query().select(["id"]).withRowId().toArray();
const rowIds = rows.map((row) => row._rowid as bigint);
const bytes = await table.fetchBlobs("video", rowIds);
const [handle] = await table.fetchBlobFiles("video", rowIds);
const size = handle!.size();
const header = await handle!.readRange(0n, size < 65536n ? size : 65536n);
```
docs/src/js/functions/connectCatalog.md
+32
View File
@@ -0,0 +1,32 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / connectCatalog
# Function: connectCatalog()
```ts
function connectCatalog(endpoint, options): Promise<Catalog>
```
Connect to an HTTP(S) catalog endpoint. Catalog requests omit database-selection
headers; opened database connections inherit authentication and client options.
## Parameters
* **endpoint**: `string`
* **options**: [`CatalogOptions`](../interfaces/CatalogOptions.md) = `{}`
## Returns
`Promise`&lt;[`Catalog`](../classes/Catalog.md)&gt;
## Example
```ts
const catalog = await connectCatalog("https://my-server.example", { apiKey: "secret" });
const db = await catalog.createDatabase("analytics", { existOk: true });
const page = await catalog.listDatabases({ limit: 20 });
```
docs/src/js/functions/isBlobField.md
+22
View File
@@ -0,0 +1,22 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / isBlobField
# Function: isBlobField()
```ts
function isBlobField(field): boolean
```
Checks for the `lance.blob.v2` extension marker. Does not validate the
field's storage type.
## Parameters
* **field**: `Field`&lt;`any`&gt;
## Returns
`boolean`
docs/src/js/functions/makeJsonField.md
+36
View File
@@ -0,0 +1,36 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / makeJsonField
# Function: makeJsonField()
```ts
function makeJsonField(name, nullable): Field
```
Create an Arrow field backed by LanceDB's JSON extension type.
## Parameters
* **name**: `string`
The field name.
* **nullable**: `boolean` = `true`
Whether the field accepts null values.
## Returns
`Field`
## Example
```ts
import { connect, makeJsonField } from "@lancedb/lancedb";
import { Schema } from "apache-arrow";
const schema = new Schema([makeJsonField("metadata")]);
const db = await connect("/path/to/database");
await db.createTable("items", [{ metadata: '{"source":"api"}' }], { schema });
```
docs/src/js/globals.md
+38 -4
View File
@@ -11,6 +11,7 @@
## Enumerations
- [ClientAuthMethod](enumerations/ClientAuthMethod.md)
- [FullTextQueryType](enumerations/FullTextQueryType.md)
- [OAuthFlowType](enumerations/OAuthFlowType.md)
- [Occur](enumerations/Occur.md)
@@ -18,20 +19,25 @@
## Classes
- [AutoQuery](classes/AutoQuery.md)
- [BlobFile](classes/BlobFile.md)
- [BooleanQuery](classes/BooleanQuery.md)
- [BoostQuery](classes/BoostQuery.md)
- [BranchContents](classes/BranchContents.md)
- [Branches](classes/Branches.md)
- [Catalog](classes/Catalog.md)
- [Connection](classes/Connection.md)
- [HeaderProvider](classes/HeaderProvider.md)
- [Index](classes/Index.md)
- [Job](classes/Job.md)
- [MakeArrowTableOptions](classes/MakeArrowTableOptions.md)
- [MatchQuery](classes/MatchQuery.md)
- [MaterializedView](classes/MaterializedView.md)
- [MergeInsertBuilder](classes/MergeInsertBuilder.md)
- [MultiMatchQuery](classes/MultiMatchQuery.md)
- [NativeJsHeaderProvider](classes/NativeJsHeaderProvider.md)
- [OAuthHeaderProvider](classes/OAuthHeaderProvider.md)
- [OAuthSession](classes/OAuthSession.md)
- [PermutationBuilder](classes/PermutationBuilder.md)
- [PhraseQuery](classes/PhraseQuery.md)
- [Query](classes/Query.md)
@@ -58,6 +64,11 @@
- [BranchDiff](interfaces/BranchDiff.md)
- [BranchIndexSummary](interfaces/BranchIndexSummary.md)
- [BranchRowCountSummary](interfaces/BranchRowCountSummary.md)
- [BucketStats](interfaces/BucketStats.md)
- [CatalogOptions](interfaces/CatalogOptions.md)
- [CherryPickError](interfaces/CherryPickError.md)
- [CherryPickPreview](interfaces/CherryPickPreview.md)
- [CherryPickResult](interfaces/CherryPickResult.md)
- [ClientConfig](interfaces/ClientConfig.md)
- [ColumnAlteration](interfaces/ColumnAlteration.md)
- [ColumnOrdering](interfaces/ColumnOrdering.md)
@@ -81,6 +92,11 @@
- [FtsToken](interfaces/FtsToken.md)
- [FullTextQuery](interfaces/FullTextQuery.md)
- [FullTextSearchOptions](interfaces/FullTextSearchOptions.md)
- [FunctionErrorFragment](interfaces/FunctionErrorFragment.md)
- [FunctionErrorRecord](interfaces/FunctionErrorRecord.md)
- [FunctionErrors](interfaces/FunctionErrors.md)
- [FunctionErrorsOptions](interfaces/FunctionErrorsOptions.md)
- [GenerationStats](interfaces/GenerationStats.md)
- [HnswPqOptions](interfaces/HnswPqOptions.md)
- [HnswSqOptions](interfaces/HnswSqOptions.md)
- [IndexConfig](interfaces/IndexConfig.md)
@@ -89,15 +105,18 @@
- [IvfFlatOptions](interfaces/IvfFlatOptions.md)
- [IvfPqOptions](interfaces/IvfPqOptions.md)
- [IvfRqOptions](interfaces/IvfRqOptions.md)
- [JobDescription](interfaces/JobDescription.md)
- [JobEventsOptions](interfaces/JobEventsOptions.md)
- [JobFailureInfo](interfaces/JobFailureInfo.md)
- [JobInfo](interfaces/JobInfo.md)
- [ListDatabasesResponse](interfaces/ListDatabasesResponse.md)
- [ListNamespacesOptions](interfaces/ListNamespacesOptions.md)
- [ListNamespacesResponse](interfaces/ListNamespacesResponse.md)
- [ListTablesOptions](interfaces/ListTablesOptions.md)
- [ListTablesResponse](interfaces/ListTablesResponse.md)
- [LsmStats](interfaces/LsmStats.md)
- [LsmWriteSpec](interfaces/LsmWriteSpec.md)
- [MergeBlocker](interfaces/MergeBlocker.md)
- [MergeBranchResult](interfaces/MergeBranchResult.md)
- [MergePreview](interfaces/MergePreview.md)
- [MaterializedViewDefinition](interfaces/MaterializedViewDefinition.md)
- [MemtableStats](interfaces/MemtableStats.md)
- [MergeResult](interfaces/MergeResult.md)
- [NativeOAuthConfig](interfaces/NativeOAuthConfig.md)
- [OAuthConfig](interfaces/OAuthConfig.md)
@@ -105,11 +124,15 @@
- [OptimizeOptions](interfaces/OptimizeOptions.md)
- [OptimizeStats](interfaces/OptimizeStats.md)
- [QueryExecutionOptions](interfaces/QueryExecutionOptions.md)
- [RefreshColumnResult](interfaces/RefreshColumnResult.md)
- [RefreshMaterializedViewResult](interfaces/RefreshMaterializedViewResult.md)
- [RemovalStats](interfaces/RemovalStats.md)
- [RenameTableOptions](interfaces/RenameTableOptions.md)
- [RestNamespaceConfig](interfaces/RestNamespaceConfig.md)
- [RetryConfig](interfaces/RetryConfig.md)
- [ScannableOptions](interfaces/ScannableOptions.md)
- [SessionLogout](interfaces/SessionLogout.md)
- [SessionStatus](interfaces/SessionStatus.md)
- [ShuffleOptions](interfaces/ShuffleOptions.md)
- [SplitCalculatedOptions](interfaces/SplitCalculatedOptions.md)
- [SplitHashOptions](interfaces/SplitHashOptions.md)
@@ -119,12 +142,14 @@
- [TableStatistics](interfaces/TableStatistics.md)
- [TimeoutConfig](interfaces/TimeoutConfig.md)
- [TlsConfig](interfaces/TlsConfig.md)
- [TokenCacheOptions](interfaces/TokenCacheOptions.md)
- [TokenResponse](interfaces/TokenResponse.md)
- [TokenizeOptions](interfaces/TokenizeOptions.md)
- [UpdateFieldMetadataResult](interfaces/UpdateFieldMetadataResult.md)
- [UpdateOptions](interfaces/UpdateOptions.md)
- [UpdateResult](interfaces/UpdateResult.md)
- [Version](interfaces/Version.md)
- [ViewDescription](interfaces/ViewDescription.md)
- [WriteExecutionOptions](interfaces/WriteExecutionOptions.md)
- [WriteProgress](interfaces/WriteProgress.md)
@@ -132,11 +157,16 @@
- [AnalyzePlanDistributedMetrics](type-aliases/AnalyzePlanDistributedMetrics.md)
- [BaseTokenizer](type-aliases/BaseTokenizer.md)
- [BlobOptions](type-aliases/BlobOptions.md)
- [BlobRange](type-aliases/BlobRange.md)
- [BlobReadOptions](type-aliases/BlobReadOptions.md)
- [Data](type-aliases/Data.md)
- [DataLike](type-aliases/DataLike.md)
- [DataTypeLike](type-aliases/DataTypeLike.md)
- [FieldLike](type-aliases/FieldLike.md)
- [IntoSql](type-aliases/IntoSql.md)
- [IntoVector](type-aliases/IntoVector.md)
- [MaterializedViewSelect](type-aliases/MaterializedViewSelect.md)
- [MultiVector](type-aliases/MultiVector.md)
- [RecordBatchLike](type-aliases/RecordBatchLike.md)
- [SchemaLike](type-aliases/SchemaLike.md)
@@ -146,10 +176,14 @@
## Functions
- [RecordBatchIterator](functions/RecordBatchIterator.md)
- [blob](functions/blob.md)
- [connect](functions/connect.md)
- [connectCatalog](functions/connectCatalog.md)
- [connectNamespace](functions/connectNamespace.md)
- [instrumentLanceDbMetrics](functions/instrumentLanceDbMetrics.md)
- [isBlobField](functions/isBlobField.md)
- [makeArrowTable](functions/makeArrowTable.md)
- [makeJsonField](functions/makeJsonField.md)
- [packBits](functions/packBits.md)
- [permutationBuilder](functions/permutationBuilder.md)
- [tokenize](functions/tokenize.md)
docs/src/js/interfaces/AddDataOptions.md
+16
View File
@@ -10,6 +10,22 @@ Options for adding data to a table.
## Properties
### allowExternalBlobOutsideBases?
```ts
optional allowExternalBlobOutsideBases: boolean;
```
Whether blob URIs outside registered external bases may be written.
Defaults to `false`. This option is supported only for local/native
tables; remote tables return an error when it is enabled. An enabled write
stores the absolute URI reference without registering a base or copying
the external object into the database. The object must remain accessible
when the blob is read later.
***
### mode
```ts
docs/src/js/interfaces/BranchDiff.md
+8 -16
View File
@@ -50,6 +50,14 @@ changedColumns: BranchColumnChange[];
***
### errors
```ts
errors: CherryPickError[];
```
***
### fromBranch
```ts
@@ -66,22 +74,6 @@ mainVersion: number;
***
### mergeBlockers
```ts
mergeBlockers: MergeBlocker[];
```
***
### mergeable
```ts
mergeable: boolean;
```
***
### parentVersion
```ts
docs/src/js/interfaces/BucketStats.md
+116
View File
@@ -0,0 +1,116 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BucketStats
# Interface: BucketStats
Live state of one bucket. A table is N buckets on one node; flattening to a
single number hides the one hot bucket that is usually why someone opened
this endpoint.
## Properties
### compacting
```ts
compacting: boolean;
```
Whether a pass owns this bucket's compaction latch right now. Says *a*
driver is running, not *whose*, and the latch is held from dispatch —
including while the pass queues for a pod-wide compactor permit. Read it
as "do not pile on", never as "mine is progressing".
***
### currentGeneration
```ts
currentGeneration: number;
```
The generation the active memtable will become.
***
### generations
```ts
generations: GenerationStats[];
```
Flushed L0 generations not yet merged into the base table.
***
### manifestVersion
```ts
manifestVersion: number;
```
Version of the shard manifest these numbers were read from.
***
### memtables?
```ts
optional memtables: MemtableStats[];
```
Oldest first, active last. Absent for a `"Sealed"` bucket, whose
in-memory state is torn down.
***
### replayAfterWalEntryPosition
```ts
replayAfterWalEntryPosition: number;
```
WAL position replay resumes from.
***
### shardId
```ts
shardId: string;
```
The shard this bucket writes.
***
### status
```ts
status: string;
```
`"Active"` or `"Sealed"` (drop-table 2PC in flight).
***
### walEntryPositionLastSeen
```ts
walEntryPositionLastSeen: number;
```
Highest WAL position the writer has seen. The difference against
`replayAfterWalEntryPosition` is the WAL lag.
***
### writerEpoch
```ts
writerEpoch: number;
```
Epoch of the writer that currently owns the shard.
docs/src/js/interfaces/CatalogOptions.md
+81
View File
@@ -0,0 +1,81 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / CatalogOptions
# Interface: CatalogOptions
Options shared by a catalog and the database connections it returns.
## Extends
- `Omit`&lt;`NativeCatalogOptions`, `"oauthConfig"`&gt;
## Properties
### apiKey?
```ts
optional apiKey: string;
```
#### Inherited from
`Omit.apiKey`
***
### clientConfig?
```ts
optional clientConfig: ClientConfig;
```
#### Inherited from
`Omit.clientConfig`
***
### headerProvider?
```ts
optional headerProvider: HeaderProvider | () => Record<string, string> | Promise<Record<string, string>>;
```
Called for each request to supply authentication headers.
***
### oauthConfig?
```ts
optional oauthConfig: OAuthConfig;
```
***
### readConsistencyInterval?
```ts
optional readConsistencyInterval: number;
```
#### Inherited from
`Omit.readConsistencyInterval`
***
### sqlHostOverride?
```ts
optional sqlHostOverride: string;
```
SQL service endpoint inherited by database connections.
#### Inherited from
`Omit.sqlHostOverride`
docs/src/js/interfaces/MergeBlocker.md → docs/src/js/interfaces/CherryPickError.md
+3 -3
View File
@@ -2,11 +2,11 @@
***
[@lancedb/lancedb](../globals.md) / MergeBlocker
[@lancedb/lancedb](../globals.md) / CherryPickError
# Interface: MergeBlocker
# Interface: CherryPickError
A reason why a branch cannot currently be merged.
A reason why a cherry-pick cannot currently land.
## Properties
docs/src/js/interfaces/CherryPickPreview.md
+17
View File
@@ -0,0 +1,17 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / CherryPickPreview
# Interface: CherryPickPreview
Changes that would be, or were, promoted by a cherry-pick.
## Properties
### promotedColumns
```ts
promotedColumns: string[];
```
docs/src/js/interfaces/MergeBranchResult.md → docs/src/js/interfaces/CherryPickResult.md
+7 -7
View File
@@ -2,11 +2,11 @@
***
[@lancedb/lancedb](../globals.md) / MergeBranchResult
[@lancedb/lancedb](../globals.md) / CherryPickResult
# Interface: MergeBranchResult
# Interface: CherryPickResult
Result of previewing or attempting a branch merge.
Result of previewing or attempting a cherry-pick.
## Properties
@@ -29,7 +29,7 @@ optional mainVersionAfter: number;
### preview
```ts
preview: MergePreview;
preview: CherryPickPreview;
```
***
@@ -38,9 +38,9 @@ preview: MergePreview;
```ts
status:
| "unknown"
| "rejected"
| "ready"
| "failed"
| "notImplemented"
| "merged";
| "cherryPicked"
| "unknown";
```
docs/src/js/interfaces/ClientConfig.md
+4
View File
@@ -22,6 +22,10 @@ optional extraHeaders: Record<string, string>;
optional idDelimiter: string;
```
The delimiter joining a namespace path and a name into one object
identifier. `"$"` is the only supported value, and leaving this unset is
how to get it; anything else is rejected when the connection is created.
***
### retryConfig?
docs/src/js/interfaces/DropNamespaceOptions.md
+1 -1
View File
@@ -21,7 +21,7 @@ Refuse to drop if non-empty (restrict) or drop recursively (cascade).
### mode?
```ts
optional mode: "fail" | "skip";
optional mode: "skip" | "fail";
```
Whether to skip if the namespace doesn't exist, or fail.
docs/src/js/interfaces/FieldMetadataUpdate.md
+2 -1
View File
@@ -17,7 +17,8 @@ metadata: Record<string, null | string>;
```
Metadata key/value pairs. Merged into the field's existing metadata by
default; a value of `null` deletes that key.
default; a value of `null` deletes that key. See
[Table.updateFieldMetadata](../classes/Table.md#updatefieldmetadata) for the conventional `lancedb:*` keys.
***
docs/src/js/interfaces/FunctionErrorFragment.md
+42
View File
@@ -0,0 +1,42 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / FunctionErrorFragment
# Interface: FunctionErrorFragment
A fragment whose per-row error detail was capped: `rowsSkipped` rows
failed, of which only `rowsRecorded` have a record of their own.
## Properties
### fragmentId
```ts
fragmentId: number;
```
***
### jobId
```ts
jobId: string;
```
***
### rowsRecorded
```ts
rowsRecorded: number;
```
***
### rowsSkipped
```ts
rowsSkipped: number;
```
docs/src/js/interfaces/FunctionErrorRecord.md
+92
View File
@@ -0,0 +1,92 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / FunctionErrorRecord
# Interface: FunctionErrorRecord
One row a Function refresh skipped, as the server recorded it.
## Properties
### column
```ts
column: string;
```
***
### createdAtMillis
```ts
createdAtMillis: number;
```
***
### errorMessage
```ts
errorMessage: string;
```
***
### errorType
```ts
errorType: string;
```
***
### fragmentId
```ts
fragmentId: number;
```
***
### function
```ts
function: string;
```
***
### functionVersion
```ts
functionVersion: string;
```
***
### jobId
```ts
jobId: string;
```
***
### rowOffset?
```ts
optional rowOffset: number;
```
The row's offset within the fragment; absent when the fragment's
detail was capped.
***
### tableVersion
```ts
tableVersion: number;
```
docs/src/js/interfaces/FunctionErrors.md
+39
View File
@@ -0,0 +1,39 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / FunctionErrors
# Interface: FunctionErrors
A table's per-row Function errors.
## Properties
### fragments
```ts
fragments: FunctionErrorFragment[];
```
Fragments whose detail was capped.
***
### records
```ts
records: FunctionErrorRecord[];
```
The recorded rows, newest job first.
***
### truncated
```ts
truncated: boolean;
```
Whether the listing stopped at its limit.
docs/src/js/interfaces/FunctionErrorsOptions.md
+39
View File
@@ -0,0 +1,39 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / FunctionErrorsOptions
# Interface: FunctionErrorsOptions
Which per-row Function errors to list; every filter is optional.
## Properties
### column?
```ts
optional column: string;
```
Only errors on this column.
***
### jobId?
```ts
optional jobId: string;
```
Only errors recorded by this job.
***
### limit?
```ts
optional limit: number;
```
At most this many records (server default 10000, cap 100000).
docs/src/js/interfaces/GenerationStats.md
+40
View File
@@ -0,0 +1,40 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / GenerationStats
# Interface: GenerationStats
One flushed L0 generation.
## Properties
### bytes
```ts
bytes: number;
```
On-disk size of the generation.
***
### generation
```ts
generation: number;
```
The generation number. Increases as memtables are sealed into L0.
***
### rows?
```ts
optional rows: number;
```
Present only when `includeGenerationRows` was requested. Off by default
because each count opens an uncached Lance dataset.
docs/src/js/interfaces/HnswPqOptions.md
+1 -1
View File
@@ -118,7 +118,7 @@ Number of sub-vectors of PQ.
This value controls how much the vector is compressed during the quantization step.
The more sub vectors there are the less the vector is compressed. The default is
the dimension of the vector divided by 16. If the dimension is not evenly divisible
by 16 we use the dimension divded by 8.
by 16 we use the dimension divided by 8.
The above two cases are highly preferred. Having 8 or 16 values per subvector allows
us to use efficient SIMD instructions.
docs/src/js/interfaces/IndexOptions.md
+1 -1
View File
@@ -16,7 +16,7 @@ optional config: Index;
Advanced index configuration
This option allows you to specify a specfic index to create and also
This option allows you to specify a specific index to create and also
allows you to pass in configuration for training the index.
See the static methods on Index for details on the various index types.
docs/src/js/interfaces/IvfPqOptions.md
+1 -1
View File
@@ -112,7 +112,7 @@ Number of sub-vectors of PQ.
This value controls how much the vector is compressed during the quantization step.
The more sub vectors there are the less the vector is compressed. The default is
the dimension of the vector divided by 16. If the dimension is not evenly divisible
by 16 we use the dimension divded by 8.
by 16 we use the dimension divided by 8.
The above two cases are highly preferred. Having 8 or 16 values per subvector allows
us to use efficient SIMD instructions.
docs/src/js/interfaces/JobDescription.md
-66
View File
@@ -1,66 +0,0 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / JobDescription
# Interface: JobDescription
A described job from `Connection.getJob`.
## Properties
### creationMs
```ts
creationMs: number;
```
When the job was created, in milliseconds since the epoch.
***
### failure?
```ts
optional failure: JobFailureInfo;
```
Why the job failed, when the job is failed and the server reports a
reason.
***
### jobId
```ts
jobId: string;
```
***
### jobType
```ts
jobType: string;
```
***
### specJson?
```ts
optional specJson: string;
```
The job-type-specific specification as a JSON string, when present.
***
### state
```ts
state: string;
```
Lifecycle state: "running", "finished", "failed", or "cancelled".
docs/src/js/interfaces/JobEventsOptions.md
+29
View File
@@ -0,0 +1,29 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / JobEventsOptions
# Interface: JobEventsOptions
Which of a job's events [Job.events](../classes/Job.md#events) returns.
## Properties
### filter?
```ts
optional filter: string;
```
SQL-like filter over the event columns.
***
### limit?
```ts
optional limit: number;
```
Maximum event rows to return, up to the server maximum of 10,000.
docs/src/js/interfaces/JobInfo.md
+1 -1
View File
@@ -26,7 +26,7 @@ When the job was created, in milliseconds since the epoch.
jobId: string;
```
The job id -- what `Connection.getJob` and `Connection.cancelJob`
The job id -- what `Connection.openJob` and `Connection.cancelJob`
accept.
***
docs/src/js/interfaces/ListDatabasesResponse.md
+23
View File
@@ -0,0 +1,23 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / ListDatabasesResponse
# Interface: ListDatabasesResponse
## Properties
### databases
```ts
databases: string[];
```
***
### pageToken?
```ts
optional pageToken: string;
```
docs/src/js/interfaces/ListTablesOptions.md
+34
View File
@@ -0,0 +1,34 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / ListTablesOptions
# Interface: ListTablesOptions
## Properties
### limit?
```ts
optional limit: number;
```
An upper bound on how many tables to return.
A page may hold fewer than this and still not be the last one, so keep
going while the response carries a page token rather than while pages are
full.
***
### pageToken?
```ts
optional pageToken: string;
```
Token from a previous response, to resume listing where it left off.
The token is opaque: it carries whatever the database needs to resume, and
callers should not construct or interpret one.
docs/src/js/interfaces/ListTablesResponse.md
+23
View File
@@ -0,0 +1,23 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / ListTablesResponse
# Interface: ListTablesResponse
## Properties
### pageToken?
```ts
optional pageToken: string;
```
***
### tables
```ts
tables: string[];
```
docs/src/js/interfaces/LsmStats.md
+22
View File
@@ -0,0 +1,22 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / LsmStats
# Interface: LsmStats
Live per-bucket LSM state, as returned by `Table#getLsmStats`.
Nothing here is derived: sums and differences (total L0 bytes, WAL lag) are
the caller's to compute.
## Properties
### buckets
```ts
buckets: BucketStats[];
```
One entry per bucket backing this table.
docs/src/js/interfaces/LsmWriteSpec.md
+3 -1
View File
@@ -34,7 +34,9 @@ Bucket and identity variants: the sharding column.
optional maintainedIndexes: string[];
```
Names of indexes the MemWAL should keep up to date during writes.
Indexes the MemWAL keeps up to date. Omit to maintain every supported
index, resolved on install — a snapshot, so indexes created later are not
maintained. Pass `[]` for none.
***
docs/src/js/interfaces/MaterializedViewDefinition.md
+22
View File
@@ -0,0 +1,22 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / MaterializedViewDefinition
# Interface: MaterializedViewDefinition
The query that defines a materialized view, as stored:
`SELECT columns FROM [ns.]table [, function(args) AS alias | , UNNEST(column) AS alias]
[WHERE predicate] [GROUP BY expr, ...] [LIMIT n]`. A Function in `FROM`
position yields one row per element it returns.
## Properties
### query
```ts
query: string;
```
The defining query, in the canonical spelling the server stores.
docs/src/js/interfaces/MemtableStats.md
+60
View File
@@ -0,0 +1,60 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / MemtableStats
# Interface: MemtableStats
One in-memory memtable.
## Properties
### batches
```ts
batches: number;
```
Record batches currently buffered.
***
### bytes
```ts
bytes: number;
```
Estimated in-memory size.
***
### generation
```ts
generation: number;
```
The generation this memtable will become once sealed.
***
### indexes
```ts
indexes: string[];
```
Names of the indexes this memtable carries. An absent name is the whole
answer to "why is my fresh-tier search on that column brute-force".
***
### rows
```ts
rows: number;
```
Rows currently buffered.
docs/src/js/interfaces/MergePreview.md
-17
View File
@@ -1,17 +0,0 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / MergePreview
# Interface: MergePreview
Changes that would be, or were, promoted by a branch merge.
## Properties
### promotedColumns
```ts
promotedColumns: string[];
```
docs/src/js/interfaces/NativeOAuthConfig.md
+76 -1
View File
@@ -15,6 +15,39 @@ All token acquisition and refresh is handled in the Rust layer.
## Properties
### audience?
```ts
optional audience: string;
```
Optional provider-specific audience for authorization and token requests.
***
### callbackPort?
```ts
optional callbackPort: number;
```
Port for the authorization_code loopback callback server.
***
### clientAuthMethod?
```ts
optional clientAuthMethod: string;
```
How the client authenticates to the token endpoint: "none",
"client_secret_basic", or "client_secret_post". Defaults to
"client_secret_basic" when a client secret is set, and "none" for
public clients.
***
### clientId
```ts
@@ -41,7 +74,8 @@ Client secret (required for client_credentials).
optional flow: string;
```
Authentication flow: "client_credentials" or "azure_managed_identity"
Authentication flow: "client_credentials", "authorization_code",
"device_code", or "azure_managed_identity"
***
@@ -66,6 +100,16 @@ Client ID for user-assigned managed identity (azure_managed_identity).
***
### redirectUri?
```ts
optional redirectUri: string;
```
Loopback redirect URI for authorization_code.
***
### refreshBufferSecs?
```ts
@@ -78,6 +122,16 @@ the TTL, each request refreshes the token.
***
### resource?
```ts
optional resource: string;
```
Optional resource indicator for authorization and token requests.
***
### scopes
```ts
@@ -86,3 +140,24 @@ scopes: string[];
OAuth scopes to request. For Azure managed identity, exactly one scope
or resource is required. For example: `["api://{app_id}/.default"]`
***
### tokenCache?
```ts
optional tokenCache: TokenCacheOptions;
```
Opt in to the persistent token cache so short-lived processes reuse
one session. Only refresh tokens are persisted.
***
### usePkce?
```ts
optional usePkce: boolean;
```
Whether authorization_code uses S256 PKCE (default: true).
docs/src/js/interfaces/OAuthConfig.md
+106
View File
@@ -26,6 +26,18 @@ const config: OAuthConfig = {
};
```
Providers requiring an explicit target can set `resource` and/or `audience`:
```typescript
const targeted: OAuthConfig = {
issuerUrl: "https://issuer.example.com",
clientId: "app-id",
clientSecret: "secret",
scopes: ["read"],
resource: "https://api.example.com",
audience: "lancedb-api",
};
```
```typescript
const config: OAuthConfig = {
issuerUrl: "https://login.microsoftonline.com/{tenant}/v2.0",
@@ -35,8 +47,57 @@ const config: OAuthConfig = {
};
```
The authorization URL is written to stderr before LanceDB tries to open a
browser, so it can be copied in headless environments.
```typescript
const config: OAuthConfig = {
issuerUrl: "https://login.microsoftonline.com/{tenant}/v2.0",
clientId: "app-id",
scopes: ["openid", "api://lancedb-api/access"],
flow: OAuthFlowType.AuthorizationCode,
};
```
Device Authorization writes the verification URL and user code to stderr
before polling begins.
## Properties
### audience?
```ts
optional audience: string;
```
Provider-specific audience, forwarded to authorization and token endpoints,
including refresh requests. Not supported for Azure managed identity.
***
### callbackPort?
```ts
optional callbackPort: number;
```
Port for the AuthorizationCode loopback callback server (default: 8400).
***
### clientAuthMethod?
```ts
optional clientAuthMethod: ClientAuthMethod;
```
How the client authenticates to the token endpoint (default: auto).
With a `clientSecret` the default is `ClientAuthMethod.ClientSecretBasic`,
which matches the RFC 6749 recommendation and the default configuration
of Okta confidential applications; without a secret the client is public
and no client authentication is sent.
***
### clientId
```ts
@@ -88,6 +149,16 @@ Client ID for user-assigned managed identity (AzureManagedIdentity).
***
### redirectUri?
```ts
optional redirectUri: string;
```
Loopback redirect URI for AuthorizationCode.
***
### refreshBufferSecs?
```ts
@@ -100,6 +171,18 @@ the TTL, each request refreshes the token.
***
### resource?
```ts
optional resource: string;
```
Resource indicator (RFC 8707), forwarded verbatim to authorization and token
endpoints, including refresh requests. Must be an absolute URI without a
fragment. Not supported for Azure managed identity.
***
### scopes
```ts
@@ -109,3 +192,26 @@ scopes: string[];
OAuth scopes to request.
For Azure managed identity, exactly one scope or resource is required.
For example: `["api://{app_id}/.default"]`
***
### tokenCache?
```ts
optional tokenCache: TokenCacheOptions;
```
Opt in to the persistent token cache so short-lived processes reuse one
session. Only refresh tokens are persisted. Only supported by
AuthorizationCode and DeviceCode; Azure managed identity is rejected.
Default: unset (memory only).
***
### usePkce?
```ts
optional usePkce: boolean;
```
Protect AuthorizationCode with S256 PKCE (default: true).
docs/src/js/interfaces/OptimizeOptions.md
+2 -1
View File
@@ -26,7 +26,8 @@ const olderThan = new Date();
olderThan.setDate(olderThan.getDate() - 1));
tbl.optimize({cleanupOlderThan: olderThan});
// Delete all versions except the current version
// Delete versions committed before this point. Versions created by the
// optimize call itself are newer than the cutoff and will be retained.
tbl.optimize({cleanupOlderThan: new Date()});
```
docs/src/js/interfaces/RefreshColumnResult.md
+23
View File
@@ -0,0 +1,23 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / RefreshColumnResult
# Interface: RefreshColumnResult
## Properties
### rowsFilled
```ts
rowsFilled: number;
```
***
### version
```ts
version: number;
```
docs/src/js/interfaces/RefreshMaterializedViewResult.md
+41
View File
@@ -0,0 +1,41 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / RefreshMaterializedViewResult
# Interface: RefreshMaterializedViewResult
## Properties
### mode
```ts
mode: string;
```
How the view was brought up to date: "rebuild", "incremental" or "no_op".
***
### rowsWritten
```ts
rowsWritten: number;
```
***
### sourceVersion
```ts
sourceVersion: number;
```
***
### version
```ts
version: number;
```
docs/src/js/interfaces/SessionLogout.md
+20
View File
@@ -0,0 +1,20 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / SessionLogout
# Interface: SessionLogout
Result of [OAuthSession.logout](../classes/OAuthSession.md#logout).
## Properties
### removed
```ts
removed: boolean;
```
Whether a cached credential was removed. `false` means no matching
session was cached; logout is idempotent.
docs/src/js/interfaces/SessionStatus.md
+94
View File
@@ -0,0 +1,94 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / SessionStatus
# Interface: SessionStatus
Safe, non-secret view of a cached OAuth session, returned by
[OAuthSession.status](../classes/OAuthSession.md#status) and [OAuthSession.login](../classes/OAuthSession.md#login).
## Properties
### audience?
```ts
optional audience: string;
```
Provider-specific audience used to obtain the cached session, if configured.
***
### clientId
```ts
clientId: string;
```
Client ID of the cached session.
***
### flow
```ts
flow: string;
```
Flow that produced the cached session.
***
### issuerUrl
```ts
issuerUrl: string;
```
Canonical issuer URL of the cached session.
***
### obtainedAt?
```ts
optional obtainedAt: number;
```
When the cached session was obtained, as Unix seconds.
***
### refreshable
```ts
refreshable: boolean;
```
Whether a cached session exists that can obtain tokens without
interactive authentication. Because access tokens are not persisted,
this is `true` exactly when a refresh token is cached; the next
connection refreshes with it rather than opening a browser or device
prompt.
***
### resource?
```ts
optional resource: string;
```
Resource indicator used to obtain the cached session, if configured.
***
### scopes
```ts
scopes: string[];
```
Canonical (sorted, de-duplicated) scope set of the cached session.
docs/src/js/interfaces/TableNamesOptions.md
+8 -3
View File
@@ -4,11 +4,16 @@
[@lancedb/lancedb](../globals.md) / TableNamesOptions
# Interface: TableNamesOptions
# Interface: ~~TableNamesOptions~~
## Deprecated
Use [ListTablesOptions](ListTablesOptions.md) with [Connection.listTables](../classes/Connection.md#listtables)
instead.
## Properties
### limit?
### ~~limit?~~
```ts
optional limit: number;
@@ -18,7 +23,7 @@ An optional limit to the number of results to return.
***
### startAfter?
### ~~startAfter?~~
```ts
optional startAfter: string;
docs/src/js/interfaces/TableStatistics.md
+4 -1
View File
@@ -44,4 +44,7 @@ The number of rows in the table
totalBytes: number;
```
The total number of bytes in the table
The total size, in bytes, of the table's data files, index files, and
overlay files
Read from the manifest, so this excludes deletion files and manifests.
docs/src/js/interfaces/TokenCacheOptions.md
+41
View File
@@ -0,0 +1,41 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / TokenCacheOptions
# Interface: TokenCacheOptions
Options for the persistent OAuth token cache.
The cache is opt-in: it is only used when set as `tokenCache` on
[OAuthConfig](OAuthConfig.md). Only refresh tokens are persisted, in a private
directory with owner-only permissions, so short-lived processes can reuse
an authenticated session instead of re-prompting on every start.
Multiple identities (issuer, client, scopes, resource, audience, flow, client authentication)
get separate cache entries. Within one identity the most recent login wins.
## Properties
### cacheDir?
```ts
optional cacheDir: string;
```
Directory that holds cached credentials. Defaults to
`$XDG_CACHE_HOME/lancedb/oauth`, `$HOME/.cache/lancedb/oauth` on Unix,
or `%LOCALAPPDATA%\\lancedb\\oauth` on Windows. The directory is created
with owner-only permissions (`0700`) when missing.
***
### lockTimeoutSecs?
```ts
optional lockTimeoutSecs: number;
```
How long to wait for the cross-process refresh lock before failing, in
seconds (default: 30).

Some files were not shown because too many files have changed in this diff Show More