rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-09-22 13:05:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into gatekeeper/fix-2325-1

Browse Source
This commit is contained in:
Gatefixer
2026-09-17 06:49:46 +00:00
parent 9bfd475702 60a1b4c219
commit 64e4d42488
221 changed files with 34375 additions and 14194 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.bumpversion.toml
+1 -1
View File
@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.38.0-beta.12"
current_version = "0.40.0-beta.1"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.
.github/dependabot.yml
+24
View File
@@ -44,3 +44,27 @@ updates:
python-deps:
patterns:
- "*"
# The npm ecosystem covers pnpm lockfiles. There are two separate installs:
# the bindings themselves and the examples, which have their own lockfile.
# As with cargo and pip above, only bump the lockfile — the version ranges
# in package.json are our consumers' constraints, not ours.
- package-ecosystem: npm
directory: /nodejs
schedule:
interval: weekly
versioning-strategy: lockfile-only
groups:
nodejs-deps:
patterns:
- "*"
- package-ecosystem: npm
directory: /nodejs/examples
schedule:
interval: weekly
versioning-strategy: lockfile-only
groups:
nodejs-examples-deps:
patterns:
- "*"
.github/workflows/dev.yml
+10 -4
View File
@@ -29,12 +29,14 @@ jobs:
steps:
- uses: actions/setup-node@v6
with:
node-version: "18"
node-version: "24"
- uses: pnpm/action-setup@v6
with:
version: 11.1.1
# These rules are disabled because Github will always ensure there
# is a blank line between the title and the body and Github will
# word wrap the description field to ensure a reasonable max line
# length.
- run: npm install @commitlint/config-conventional
- run: >
echo 'module.exports = {
"rules": {
@@ -43,7 +45,11 @@ jobs:
"body-leading-blank": [0, "always"]
}
}' > .commitlintrc.js
- run: npx commitlint --extends @commitlint/config-conventional --verbose <<< $COMMIT_MSG
- run: >
pnpm dlx
--package @commitlint/cli@21.2.2
--package @commitlint/config-conventional@21.2.2
commitlint --extends @commitlint/config-conventional --verbose <<< $COMMIT_MSG
env:
COMMIT_MSG: >
${{ github.event.pull_request.title }}
@@ -54,7 +60,7 @@ jobs:
with:
script: |
const message = `**ACTION NEEDED**
Lance follows the [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0/) for release automation.
The PR title and description are used as the merge commit message.\
.github/workflows/docs-link-check.yml
+1 -1
View File
@@ -56,7 +56,7 @@ jobs:
uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2.9.0
with:
# Restricted to http(s) on purpose. Much of docs/src is generated
# API reference (the js/ tree comes from `npm run docs` in nodejs)
# API reference (the js/ tree comes from `pnpm run docs` in nodejs)
# and the hand-written pages use mkdocstrings cross-references and
# nav-relative paths that only resolve in the site mkdocs builds,
# not in this checkout, so relative links would be reported as
.github/workflows/docs.yml
+1 -3
View File
@@ -55,9 +55,7 @@ jobs:
- name: Set up node
uses: actions/setup-node@v6
with:
node-version: 20
cache: 'npm'
cache-dependency-path: docs/package-lock.json
node-version: 24
- name: Install node dependencies
working-directory: nodejs
run: |
.github/workflows/nodejs.yml
+12 -14
View File
@@ -47,9 +47,8 @@ jobs:
version: 11.1.1
- uses: actions/setup-node@v6
with:
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
# in October. The library itself still supports Node >= 18
# (see test matrix below).
# Build on a supported LTS; the matrix job below covers every
# Node version the library claims to support.
node-version: 24
cache: 'pnpm'
cache-dependency-path: nodejs/pnpm-lock.yaml
@@ -84,7 +83,7 @@ jobs:
timeout-minutes: 30
strategy:
matrix:
node-version: [ "18", "20" ]
node-version: [ "22", "24", "26" ]
runs-on: "ubuntu-22.04"
defaults:
run:
@@ -101,9 +100,9 @@ jobs:
- uses: actions/setup-node@v6
name: Setup Node.js 24 for build
with:
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
# in October. Build/install runs on Node 24; tests run on the
# matrix version below using direct jest invocation.
# Build and install once on a fixed version so the generated docs
# are identical across matrix legs; the tests below then run on each
# supported Node version.
node-version: 24
cache: 'pnpm'
cache-dependency-path: nodejs/pnpm-lock.yaml
@@ -152,9 +151,9 @@ jobs:
S3_TEST: "1"
# Newer @smithy/core uses dynamic ESM imports.
NODE_OPTIONS: "--experimental-vm-modules"
# Invoke jest directly because pnpm 11 itself requires Node 22+
# while the matrix tests on older Node versions.
run: npx jest --verbose
# Invoke the installed jest binary directly; the pnpm shim is set up
# against the build-phase Node, not the version selected above.
run: node_modules/.bin/jest --verbose
- name: Test examples
working-directory: ./
env:
@@ -164,7 +163,7 @@ jobs:
run: |
python ci/mock_openai.py &
cd nodejs/examples
npx jest --testEnvironment jest-environment-node-single-context --verbose
node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose
macos:
timeout-minutes: 30
# macos-15 ships a newer linker; the older macos-14 linker fails to insert
@@ -185,8 +184,7 @@ jobs:
version: 11.1.1
- uses: actions/setup-node@v6
with:
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
# in October.
# pnpm 11 requires Node >= 22.13.
node-version: 24
cache: 'pnpm'
cache-dependency-path: nodejs/pnpm-lock.yaml
@@ -209,4 +207,4 @@ jobs:
pnpm tsc
- name: Test
run: |
pnpm test
pnpm test --runInBand
.github/workflows/npm-publish.yml
+7 -8
View File
@@ -168,8 +168,7 @@ jobs:
- name: Setup node
uses: actions/setup-node@v6
with:
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
# in October.
# pnpm 11 requires Node >= 22.13.
node-version: 24
cache: pnpm
cache-dependency-path: nodejs/pnpm-lock.yaml
@@ -251,7 +250,7 @@ jobs:
run: |
set -e
${{ matrix.settings.pre_build }}
npx napi build --platform --release \
node_modules/.bin/napi build --platform --release \
--features ${{ matrix.settings.features }} \
--target ${{ matrix.settings.target }} \
--dts ../lancedb/native.d.ts \
@@ -271,7 +270,7 @@ jobs:
- name: Build
run: |
${{ matrix.settings.pre_build }}
npx napi build --platform --release \
node_modules/.bin/napi build --platform --release \
--features ${{ matrix.settings.features }} \
--target ${{ matrix.settings.target }} \
--dts ../lancedb/native.d.ts \
@@ -339,7 +338,7 @@ jobs:
- target: aarch64-unknown-linux-gnu
host: ubuntu-2404-8x-arm64
node:
- '20'
- '22'
runs-on: ${{ matrix.settings.host }}
defaults:
run:
@@ -385,9 +384,9 @@ jobs:
- name: Move built files
run: cp dist/native.d.ts dist/native.js dist/*.node lancedb/
- name: Test bindings
# Invoke jest directly because pnpm 11 itself requires Node 22+
# while the matrix tests on older Node versions.
run: npx jest --verbose
# Invoke the installed jest binary directly; the pnpm shim is set up
# against the install-phase Node, not the version selected above.
run: node_modules/.bin/jest --verbose
publish:
name: Publish
runs-on: ubuntu-latest
.github/workflows/python.yml
+1 -1
View File
@@ -156,7 +156,7 @@ jobs:
- name: Test without pylance or pandas
run: |
pip uninstall -y pylance pandas
pytest -vv python/tests/test_table.py
pytest -vv python/tests/test_table.py python/tests/test_namespace_no_pylance.py
# Make sure wheels are not included in the Rust cache
- name: Delete wheels
run: rm -rf target/wheels
.github/workflows/rust.yml
+4 -1
View File
@@ -232,7 +232,10 @@ jobs:
ALL_FEATURES=`cargo metadata --format-version=1 --no-deps \
| jq -r '.packages[] | .features | keys | .[]' \
| grep -v s3-test | sort | uniq | paste -s -d "," -`
cargo test --profile ci --features $ALL_FEATURES --locked
# Run doctests before test binaries fill the runner disk. Examples are
# already built by the Linux job, so avoid retaining them here.
cargo test --profile ci --features $ALL_FEATURES --locked --doc
cargo test --profile ci --features $ALL_FEATURES --locked --lib --tests
windows:
strategy:
.github/workflows/typos.yml
+20
View File
@@ -0,0 +1,20 @@
name: Typo checker
on:
push:
branches:
- main
pull_request:
permissions:
contents: read
jobs:
run:
name: Spell Check with Typos
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Check spelling of the entire repository
uses: crate-ci/typos@6802cc60d4e7f78b9d5454f6cf3935c042d5e1e3 # v1.26.0
.github/workflows/update_package_lock_run.yml
-22
View File
@@ -1,22 +0,0 @@
name: Update package-lock.json
on:
workflow_dispatch:
permissions:
contents: read
jobs:
publish:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
with:
ref: main
persist-credentials: false
fetch-depth: 0
lfs: true
- uses: ./.github/workflows/update_package_lock
with:
github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
.github/workflows/update_package_lock_run_nodejs.yml
-22
View File
@@ -1,22 +0,0 @@
name: Update NodeJs package-lock.json
on:
workflow_dispatch:
permissions:
contents: read
jobs:
publish:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
with:
ref: main
persist-credentials: false
fetch-depth: 0
lfs: true
- uses: ./.github/workflows/update_package_lock_nodejs
with:
github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
.pre-commit-config.yaml
+8 -1
View File
@@ -10,6 +10,10 @@ repos:
rev: v0.9.9
hooks:
- id: ruff
- repo: https://github.com/crate-ci/typos
rev: v1.26.0
hooks:
- id: typos
# - repo: https://github.com/RobertCraigie/pyright-python
# rev: v1.1.395
# hooks:
@@ -20,7 +24,10 @@ repos:
hooks:
- id: local-biome-check
name: biome check
entry: npx @biomejs/biome@1.8.3 check --config-path nodejs/biome.json nodejs/
# Use the biome from nodejs/package.json rather than a separately
# pinned one: the two drifted apart and disagreed on formatting, so
# this hook rejected code that `pnpm lint` accepted.
entry: nodejs/node_modules/.bin/biome check --config-path nodejs/biome.json nodejs/
language: system
types: [text]
files: "nodejs/.*"
.typos.toml
+19
View File
@@ -0,0 +1,19 @@
[default]
extend-ignore-re = ["(?Rm)^.*(#|//)\\s*spellchecker:disable-line$"]
[default.extend-words]
# Azure Kubernetes Service, mentioned in rust/lancedb/src/remote/oauth.rs.
AKS = "AKS"
# RabitQ is the name of a vector quantization algorithm, not a typo of "Rabbit".
Rabit = "Rabit"
# `VarBuilder::from_mmaped_safetensors` is the real (if oddly-spelled) name of
# the candle-core API we call in rust/lancedb/src/embeddings/sentence_transformers.rs.
mmaped = "mmaped"
# `WriteableBuffer` is the real name of a type from Python's `_typeshed` stubs,
# used in python/python/lancedb/_blob.py.
Writeable = "Writeable"
[files]
extend-exclude = [
"*_THIRD_PARTY_LICENSES.*",
]
AGENTS.md
+3 -3
View File
@@ -38,7 +38,7 @@ Before committing changes, run formatting for every language you touched. At min
* Rust changes: run `cargo fmt --all`.
* Python changes: run `ruff format .` and `ruff check .` from the repository root,
and run targeted tests through `cd python && uv run ...`.
* TypeScript changes: run the relevant `npm`/`pnpm` lint, format, build, and docs commands in `nodejs`.
* TypeScript changes: run the relevant `pnpm` lint, format, build, and docs commands in `nodejs`.
Before creating a PR, the exact value passed to `gh pr create --title` must follow
Conventional Commits, such as `fix: support nested field paths in native index creation`
@@ -101,12 +101,12 @@ Python bindings changes:
TypeScript bindings changes:
1. Add napi-rs method binding on `Table` in `nodejs/src/table.rs`.
2. Run `npm run build` to generate TypeScript definitions.
2. Run `pnpm build` to generate TypeScript definitions.
3. Add typescript method on abstract class `Table` in `nodejs/src/table.ts`.
4. Add concrete method on `LocalTable` class in `nodejs/src/native_table.ts`.
* Note: despite the name, this class is also used for remote tables.
5. Add test in `nodejs/__test__/table.test.ts`.
6. Run `npm run docs` to generate TypeScript documentation.
6. Run `pnpm run docs` to generate TypeScript documentation.
## Python API reference
Cargo.lock
Generated
+804 -316
View File
File diff suppressed because it is too large Load Diff
Cargo.toml
+21 -16
View File
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
rust-version = "1.91.0"
[workspace.dependencies]
lance = { "version" = "=12.0.0-beta.2", default-features = false, "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=12.0.0-beta.2", default-features = false, "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=12.0.0-beta.2", default-features = false, "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=13.0.0-beta.4", default-features = false, "tag" = "v13.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=13.0.0-beta.4", "tag" = "v13.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=13.0.0-beta.4", "tag" = "v13.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=13.0.0-beta.4", "tag" = "v13.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=13.0.0-beta.4", default-features = false, "tag" = "v13.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=13.0.0-beta.4", "tag" = "v13.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=13.0.0-beta.4", "tag" = "v13.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=13.0.0-beta.4", "tag" = "v13.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=13.0.0-beta.4", default-features = false, "tag" = "v13.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=13.0.0-beta.4", "tag" = "v13.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=13.0.0-beta.4", "tag" = "v13.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=13.0.0-beta.4", "tag" = "v13.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=13.0.0-beta.4", "tag" = "v13.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=13.0.0-beta.4", "tag" = "v13.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
lancedb = { path = "rust/lancedb", default-features = false }
ahash = "0.8"
# Note that this one does not include pyarrow
@@ -39,7 +39,10 @@ arrow-ord = "58.0.0"
arrow-schema = "58.0.0"
arrow-select = "58.0.0"
arrow-cast = "58.0.0"
arrow-flight = { version = "58.0.0", features = ["flight-sql-experimental"] }
async-trait = "0"
# Smithy JSON 0.63 requires the pre-1.7 Document representation; allow the MSRV pin.
aws-smithy-types = ">=1.3.6, <1.7"
bytes = "1"
datafusion = { version = "54.0.0", default-features = false }
datafusion-catalog = "54.0.0"
@@ -59,19 +62,21 @@ log = "0.4"
metrics = "0.24"
metrics-util = "0.19"
moka = { version = "0.12", features = ["future"] }
object_store = "0.13.2"
object_store = "0.14.1"
pin-project = "1.0.7"
rand = "0.9"
snafu = "0.8"
url = "2"
num-traits = "0.2"
oauth2 = { version = "5.0", default-features = false }
regex = "1.10"
semver = "1.0.25"
serde = "1"
serde_json = "1"
tempfile = "3.5.0"
tokio = { version = "1.23", features = ["rt-multi-thread", "sync"] }
uuid = { version = "1.7.0", features = ["v4"] }
tonic = { version = "0.14", features = ["tls-native-roots", "tls-ring"] }
uuid = { version = "1.7.0", features = ["v4", "v7"] }
chrono = { version = "0.4", default-features = false, features = ["clock"] }
[profile.ci]
Makefile
+1 -1
View File
@@ -5,5 +5,5 @@ licenses:
cd python && cargo about generate ../about.hbs -o RUST_THIRD_PARTY_LICENSES.html -c ../about.toml
cd python && uv sync --all-extras && uv tool run pip-licenses --python .venv/bin/python --format=markdown --with-urls --output-file=PYTHON_THIRD_PARTY_LICENSES.md
cd nodejs && cargo about generate ../about.hbs -o RUST_THIRD_PARTY_LICENSES.html -c ../about.toml
cd nodejs && npx license-checker --markdown --out NODEJS_THIRD_PARTY_LICENSES.md
cd nodejs && pnpm dlx license-checker@25 --markdown --out NODEJS_THIRD_PARTY_LICENSES.md
cd java && ./mvnw license:aggregate-add-third-party -q
ci/update_lockfiles.sh
+2 -6
View File
@@ -12,16 +12,12 @@ done
# This updates the lockfile without building
cargo metadata --quiet > /dev/null
pushd nodejs || exit 1
npm install --package-lock-only --silent
popd
if git diff --quiet --exit-code; then
echo "No lockfile changes to commit; skipping amend."
elif $AMEND; then
git add Cargo.lock nodejs/package-lock.json
git add Cargo.lock
git commit --amend --no-edit
else
git add Cargo.lock nodejs/package-lock.json
git add Cargo.lock
git commit -m "Update lockfiles"
fi
docs/README.md
+11 -8
View File
@@ -47,22 +47,24 @@ pytest -vv python/tests/docs
### Checking typescript examples
The `@lancedb/lancedb` package must be built before running the tests:
The examples depend on `@lancedb/lancedb` at `file:../dist`, so the package must be
built before running the tests. This uses pnpm; see the
[Typescript contributing guide](../nodejs/CONTRIBUTING.md) for the toolchain setup.
```shell
pushd nodejs
npm ci
npm run build
pnpm install
pnpm build
popd
```
Then you can run the examples by going to the `nodejs/examples` directory and
running the tests like a normal npm package:
Then you can run the examples by going to the `nodejs/examples` directory, which is a
separate pnpm package with its own lockfile:
```shell
pushd nodejs/examples
npm ci
npm test
pnpm install
pnpm test
popd
```
@@ -84,6 +86,7 @@ The new files should be checked into the repository.
```shell
pushd nodejs
npm run docs
# `pnpm docs` would invoke pnpm's built-in `docs` command, not the script.
pnpm run docs
popd
```
docs/openapi.yml
+10 -1
View File
@@ -155,7 +155,7 @@ paths:
vector:
type: FixedSizeList
description: |
The targetted vector to search for. Required.
The targeted vector to search for. Required.
vector_column:
type: string
description: |
@@ -446,6 +446,15 @@ paths:
properties:
column:
type: string
name:
type: string
description: Optional name for the created index.
replace:
type: boolean
default: true
description: |
Whether to replace an existing index with the same resolved
name. Defaults to true.
metric_type:
type: string
nullable: false
docs/package-lock.json
Generated
-135
View File
@@ -1,135 +0,0 @@
{
"name": "lancedb-docs-test",
"version": "1.0.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "lancedb-docs-test",
"version": "1.0.0",
"license": "Apache 2",
"dependencies": {
"apache-arrow": "file:../node/node_modules/apache-arrow",
"vectordb": "file:../node"
},
"devDependencies": {
"@types/node": "^20.11.8",
"typescript": "^5.3.3"
}
},
"../node": {
"name": "vectordb",
"version": "0.21.2-beta.0",
"cpu": [
"x64",
"arm64"
],
"license": "Apache-2.0",
"os": [
"darwin",
"linux",
"win32"
],
"dependencies": {
"@neon-rs/load": "^0.0.74",
"axios": "^1.4.0"
},
"devDependencies": {
"@neon-rs/cli": "^0.0.160",
"@types/chai": "^4.3.4",
"@types/chai-as-promised": "^7.1.5",
"@types/mocha": "^10.0.1",
"@types/node": "^18.16.2",
"@types/sinon": "^10.0.15",
"@types/temp": "^0.9.1",
"@types/uuid": "^9.0.3",
"@typescript-eslint/eslint-plugin": "^5.59.1",
"apache-arrow-old": "npm:apache-arrow@13.0.0",
"cargo-cp-artifact": "^0.1",
"chai": "^4.3.7",
"chai-as-promised": "^7.1.1",
"eslint": "^8.39.0",
"eslint-config-standard-with-typescript": "^34.0.1",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-n": "^15.7.0",
"eslint-plugin-promise": "^6.1.1",
"mocha": "^10.2.0",
"openai": "^4.24.1",
"sinon": "^15.1.0",
"temp": "^0.9.4",
"ts-node": "^10.9.1",
"ts-node-dev": "^2.0.0",
"typedoc": "^0.24.7",
"typedoc-plugin-markdown": "^3.15.3",
"typescript": "^5.1.0",
"uuid": "^9.0.0"
},
"optionalDependencies": {
"@lancedb/vectordb-darwin-arm64": "0.21.2-beta.0",
"@lancedb/vectordb-darwin-x64": "0.21.2-beta.0",
"@lancedb/vectordb-linux-arm64-gnu": "0.21.2-beta.0",
"@lancedb/vectordb-linux-x64-gnu": "0.21.2-beta.0",
"@lancedb/vectordb-win32-x64-msvc": "0.21.2-beta.0"
},
"peerDependencies": {
"@apache-arrow/ts": "^14.0.2",
"apache-arrow": "^14.0.2"
}
},
"../node/node_modules/apache-arrow": {
"version": "14.0.2",
"license": "Apache-2.0",
"dependencies": {
"@types/command-line-args": "5.2.0",
"@types/command-line-usage": "5.0.2",
"@types/node": "20.3.0",
"@types/pad-left": "2.1.1",
"command-line-args": "5.2.1",
"command-line-usage": "7.0.1",
"flatbuffers": "23.5.26",
"json-bignum": "^0.0.3",
"pad-left": "^2.1.0",
"tslib": "^2.5.3"
},
"bin": {
"arrow2csv": "bin/arrow2csv.js"
}
},
"node_modules/@types/node": {
"version": "20.11.8",
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.8.tgz",
"integrity": "sha512-i7omyekpPTNdv4Jb/Rgqg0RU8YqLcNsI12quKSDkRXNfx7Wxdm6HhK1awT3xTgEkgxPn3bvnSpiEAc7a7Lpyow==",
"dev": true,
"dependencies": {
"undici-types": "~5.26.4"
}
},
"node_modules/apache-arrow": {
"resolved": "../node/node_modules/apache-arrow",
"link": true
},
"node_modules/typescript": {
"version": "5.3.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz",
"integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==",
"dev": true,
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
},
"engines": {
"node": ">=14.17"
}
},
"node_modules/undici-types": {
"version": "5.26.5",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
"integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
"dev": true
},
"node_modules/vectordb": {
"resolved": "../node",
"link": true
}
}
}
docs/package.json
-20
View File
@@ -1,20 +0,0 @@
{
"name": "lancedb-docs-test",
"version": "1.0.0",
"description": "auto-generated tests from doc",
"author": "dev@lancedb.com",
"license": "Apache 2",
"dependencies": {
"apache-arrow": "file:../node/node_modules/apache-arrow",
"vectordb": "file:../node"
},
"scripts": {
"build": "tsc -b && cd ../node && npm run build-release",
"example": "npm run build && node",
"test": "npm run build && ls dist/*.js | xargs -n 1 node"
},
"devDependencies": {
"@types/node": "^20.11.8",
"typescript": "^5.3.3"
}
}
docs/src/java/java.md
+1 -1
View File
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
<dependency>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-core</artifactId>
<version>0.38.0-beta.12</version>
<version>0.40.0-beta.1</version>
</dependency>
```
docs/src/js/classes/BlobFile.md
+62
View File
@@ -0,0 +1,62 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobFile
# Class: BlobFile
A lazy handle to blob bytes. Create one with [Table.fetchBlobFiles](Table.md#fetchblobfiles).
## Methods
### read()
```ts
read(): Promise<Buffer>
```
Reads from the cursor to the end and advances the cursor.
A second call returns an empty buffer. [BlobFile.readRange](BlobFile.md#readrange) does
not move the cursor.
#### Returns
`Promise`&lt;`Buffer`&gt;
***
### readRange()
```ts
readRange(start, end): Promise<Buffer>
```
Reads the half-open byte range `[start, end)`.
Fails when `end` is past the blob size. Does not move the cursor.
#### Parameters
* **start**: `bigint`
* **end**: `bigint`
#### Returns
`Promise`&lt;`Buffer`&gt;
***
### size()
```ts
size(): bigint
```
Returns the blob size in bytes.
#### Returns
`bigint`
docs/src/js/classes/Catalog.md
+107
View File
@@ -0,0 +1,107 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / Catalog
# Class: Catalog
A remote catalog manages databases through the server's root namespace.
## Accessors
### uri
```ts
get uri(): string
```
The root namespace endpoint.
#### Returns
`string`
## Methods
### connectDatabase()
```ts
connectDatabase(name): Promise<Connection>
```
Connect to an existing database by its logical name.
#### Parameters
* **name**: `string`
#### Returns
`Promise`&lt;[`Connection`](Connection.md)&gt;
***
### createDatabase()
```ts
createDatabase(name, options): Promise<Connection>
```
Create a database, or open an existing database when existOk is true.
#### Parameters
* **name**: `string`
* **options** = `{}`
* **options.existOk?**: `boolean`
#### Returns
`Promise`&lt;[`Connection`](Connection.md)&gt;
***
### dropDatabase()
```ts
dropDatabase(name, options): Promise<void>
```
Drop an empty database. The server rejects nonempty databases.
#### Parameters
* **name**: `string`
* **options** = `{}`
* **options.ignoreMissing?**: `boolean`
#### Returns
`Promise`&lt;`void`&gt;
***
### listDatabases()
```ts
listDatabases(options): Promise<ListDatabasesResponse>
```
List one page of databases; pass pageToken from a response for the next page.
#### Parameters
* **options** = `{}`
* **options.limit?**: `number`
* **options.pageToken?**: `string`
#### Returns
`Promise`&lt;[`ListDatabasesResponse`](../interfaces/ListDatabasesResponse.md)&gt;
docs/src/js/classes/Connection.md
+79 -67
View File
@@ -180,13 +180,13 @@ abstract createMaterializedView(
Define a materialized view named `name` over the table `source`.
The view is created empty, with the query recorded in its schema
metadata; `view.refresh()` computes the rows. The view is a normal
table: it can be queried, indexed and searched, and it appears in
`tableNames`. The source table must have stable row ids (create it with
The view is populated before creation returns. Set `withNoData` to create
only its definition and empty backing table. The view is a normal table:
it can be queried, indexed and searched, and it appears in `tableNames`.
The source table must have stable row ids (create it with
the `newTableEnableStableRowIds` storage option); they keep the view's
provenance valid across source compactions and cannot be enabled after
a table exists. Local databases only.
a table exists.
#### Parameters
@@ -202,6 +202,8 @@ a table exists. Local databases only.
* **options.where?**: `string`
* **options.withNoData?**: `boolean`
#### Returns
`Promise`&lt;[`MaterializedView`](MaterializedView.md)&gt;
@@ -373,6 +375,54 @@ Drop all tables in the database.
***
### dropMaterializedView()
```ts
abstract dropMaterializedView(name, namespacePath?): Promise<void>
```
Drop the materialized view named `name`.
The view may become unavailable before physical cleanup finishes. Use
[dropMaterializedViewAsync](Connection.md#dropmaterializedviewasync) to retain and wait for the cleanup job.
Rejects a table that exists but is not a materialized view.
#### Parameters
* **name**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;`void`&gt;
***
### dropMaterializedViewAsync()
```ts
abstract dropMaterializedViewAsync(name, namespacePath?): Promise<Job>
```
Start dropping the materialized view named `name` and return its cleanup
job without waiting for completion.
Rejects a table that exists but is not a materialized view.
#### Parameters
* **name**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;[`Job`](Job.md)&gt;
***
### dropNamespace()
```ts
@@ -448,26 +498,6 @@ on the returned job to know when cleanup has finished.
***
### getJob()
```ts
abstract getJob(jobId): Promise<null | JobDescription>
```
Describe a single server-side job by id.
Resolves to `null` when the server has no such job.
#### Parameters
* **jobId**: `string`
#### Returns
`Promise`&lt;`null` \| [`JobDescription`](../interfaces/JobDescription.md)&gt;
***
### isOpen()
```ts
@@ -482,48 +512,6 @@ Return true if the connection has not been closed
***
### job()
```ts
abstract job(jobId): Job
```
A [Job](Job.md) handle for a server-side job by id.
The handle is constructed without a server round trip; an unknown id
surfaces when the handle is used. Dropping the handle has no effect on
the job itself.
#### Parameters
* **jobId**: `string`
#### Returns
[`Job`](Job.md)
***
### jobHistory()
```ts
abstract jobHistory(jobId?): Promise<Table<any>>
```
The lifecycle event history of a server-side job, as an Arrow table.
Lists history across all jobs when `jobId` is omitted.
#### Parameters
* **jobId?**: `string`
#### Returns
`Promise`&lt;`Table`&lt;`any`&gt;&gt;
***
### listJobs()
```ts
@@ -648,6 +636,30 @@ A page of table names and an
***
### openJob()
```ts
abstract openJob(jobId): Promise<Job>
```
Open a server-side job by id, returning a handle with its record already
populated. Rejects when the server has no such job, the way
[Connection.openTable](Connection.md#opentable) does for a missing table.
The returned [Job](Job.md) answers for its own state, specification,
result, failure and event history, so there is no separate
connection-level call for any of them.
#### Parameters
* **jobId**: `string`
#### Returns
`Promise`&lt;[`Job`](Job.md)&gt;
***
### openMaterializedView()
```ts
docs/src/js/classes/Job.md
+159 -12
View File
@@ -8,19 +8,46 @@
A handle to an operation that may still be running.
## Constructors
The operation may already be complete when the handle is created.
### new Job()
The detail getters read what the handle last observed. Submitting an
operation returns only a job id, so populating them eagerly would cost an
extra round trip on every call:
- [Job.refresh](Job.md#refresh) and [Job.status](Job.md#status) fetch the whole record.
- [Job.wait](Job.md#wait) records the terminal state it establishes, but not the
rest of the record.
- Everything is null until one of those runs.
## Accessors
### creationMs
```ts
new Job(): Job
get creationMs(): null | number
```
When the job was created, in milliseconds since the epoch.
#### Returns
[`Job`](Job.md)
`null` \| `number`
## Accessors
***
### failure
```ts
get failure(): null | JobFailureInfo
```
Why the job failed, when it failed and the server reports a reason.
#### Returns
`null` \| [`JobFailureInfo`](../interfaces/JobFailureInfo.md)
***
### id
@@ -28,8 +55,69 @@ new Job(): Job
get id(): null | string
```
Identifies the operation on the server that is running it. Operations
that run in this process have no server id. The value is opaque.
Identifies the operation on the server that is running it.
Operations that run in this process have no server id. The value is
opaque: parsing it or storing it to resume the job later is not supported.
#### Returns
`null` \| `string`
***
### jobType
```ts
get jobType(): null | string
```
The job's type, as the server names it. Null for an in-process job, which
has no server-side record.
#### Returns
`null` \| `string`
***
### result
```ts
get result(): any
```
The job-type-specific terminal result. Null until the job succeeds, so a
job that never terminates reports its progress through [Job.events](Job.md#events)
instead.
#### Returns
`any`
***
### spec
```ts
get spec(): any
```
The job-type-specific specification it was submitted with.
#### Returns
`any`
***
### state
```ts
get state(): null | string
```
The last observed lifecycle state, without contacting the backend.
#### Returns
@@ -51,18 +139,61 @@ Request cancellation. Cancelling a finished operation is a no-op.
***
### events()
```ts
events(options?): Promise<Table<any>>
```
This job's recorded lifecycle events.
Where the getters above report a terminal result only once the job reaches
one, events are written as the job runs and outlive the workers that
produced them. A distributed job records a `claim`/`claim_complete` pair
per unit of work, each carrying `rows_processed`, so a job that never
finishes still accounts for what it did.
The server caps results at 1000 rows by default and 10,000 at most, and
truncates without saying so, so pass `limit` for a job that emits an event
per fragment. `filter` is a SQL-like expression over the `state`,
`updated_by`, `emitted_from`, `emitted_by`, and `claim_entity` columns.
#### Parameters
* **options?**: [`JobEventsOptions`](../interfaces/JobEventsOptions.md)
#### Returns
`Promise`&lt;`Table`&lt;`any`&gt;&gt;
***
### refresh()
```ts
refresh(): Promise<void>
```
Ask the backend for this job's current state, and for a server-side job
its full record, then cache it for the getters above.
#### Returns
`Promise`&lt;`void`&gt;
***
### status()
```ts
status(): Promise<string>
```
The operation's current lifecycle state: "running", "finished",
"failed", or "cancelled".
The operation's current lifecycle state: "running", "finished", "failed",
or "cancelled".
A point snapshot; unlike [Job.wait](Job.md#wait) it does not block or reject
on a terminal failure state. States a newer server reports that this
client version does not know pass through as-is.
A point snapshot; unlike [Job.wait](Job.md#wait) it does not block or reject on a
terminal failure state. Also refreshes the getters above.
#### Returns
@@ -70,6 +201,22 @@ client version does not know pass through as-is.
***
### toString()
```ts
toString(): string
```
Every field the handle currently knows, one per line, with the JSON
payloads indented -- a refresh job's spec and result are the point of
printing it.
#### Returns
`string`
***
### wait()
```ts
docs/src/js/classes/MaterializedView.md
+1 -1
View File
@@ -49,7 +49,7 @@ get name(): string
definition(): Promise<MaterializedViewDefinition>
```
The query that defines the view, read from its stored schema.
The query that defines the view.
#### Returns
docs/src/js/classes/MergeInsertBuilder.md
+1 -1
View File
@@ -141,7 +141,7 @@ Currently this causes multiple copies of the row to be created
but that behavior is subject to change.
An optional condition may be specified. If it is, then only
matched rows that satisfy the condtion will be updated. Any
matched rows that satisfy the condition will be updated. Any
rows that do not satisfy the condition will be left as they
are. Failing to satisfy the condition does not cause a
"matched row" to become a "not matched" row.
docs/src/js/classes/OAuthSession.md
+105
View File
@@ -0,0 +1,105 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / OAuthSession
# Class: OAuthSession
Explicit OAuth session lifecycle for the persistent token cache: eager
`login`, non-secret `status`, and local `logout`.
A session is built from the same [OAuthConfig](../interfaces/OAuthConfig.md) used to connect
(including its `tokenCache` options). A connection created with the same
configuration shares the cache, so logging in here prepares tokens for
later processes without any database request.
`login` always runs the configured interactive flow and replaces the cached
session (the most recent login wins). `logout` removes only the local
credential; it does not revoke anything with the provider and does not sign
out of a browser SSO session.
## Example
```typescript
const config: OAuthConfig = {
issuerUrl: "https://issuer.example.com",
clientId: "my-app",
scopes: ["openid", "offline_access"],
flow: OAuthFlowType.DeviceCode,
tokenCache: { cacheDir: "/tmp/my-app/oauth-cache" },
};
const session = new OAuthSession(config);
const status = await session.login();
```
## Constructors
### new OAuthSession()
```ts
new OAuthSession(config): OAuthSession
```
Create a session manager for the given OAuth configuration.
#### Parameters
* **config**: [`OAuthConfig`](../interfaces/OAuthConfig.md)
#### Returns
[`OAuthSession`](OAuthSession.md)
## Methods
### login()
```ts
login(): Promise<SessionStatus>
```
Eagerly run the configured authentication flow and store the session.
A successful login always replaces any prior cached session for this
identity; if the provider does not issue a refresh token (for example
without `offline_access`), the previous record is removed and the status
reports `refreshable == false`.
#### Returns
`Promise`&lt;[`SessionStatus`](../interfaces/SessionStatus.md)&gt;
***
### logout()
```ts
logout(): Promise<SessionLogout>
```
Remove the matching local cached credential.
This only deletes the local cache entry. It does not revoke the refresh
token with the provider and does not sign out of a browser SSO session.
Repeated calls succeed; `removed` reports whether a credential existed.
#### Returns
`Promise`&lt;[`SessionLogout`](../interfaces/SessionLogout.md)&gt;
***
### status()
```ts
status(): Promise<SessionStatus>
```
Report whether a matching cached session exists, with safe metadata.
This never contacts the identity provider and never exposes token values.
#### Returns
`Promise`&lt;[`SessionStatus`](../interfaces/SessionStatus.md)&gt;
docs/src/js/classes/Table.md
+79 -9
View File
@@ -74,10 +74,10 @@ now: the column is committed with no values, and rows get them from
[Table#refreshColumn](Table.md#refreshcolumn). Declaring one therefore costs the same on a
large table as on an empty one.
A refresh does not revisit rows it has already filled, so mutating an
input leaves the value computed at fill time; recomputing means dropping
the column and declaring it again. While a declaration reads a column,
that column cannot be renamed, retyped or dropped.
A refresh also recomputes the rows whose inputs changed since they were
computed, so a mutated input is reflected by the next refresh. While a
declaration reads a column, that column cannot be renamed, retyped or
dropped.
On LanceDB Cloud and Enterprise the expression is planned by the
server, and the refresh runs as a server job -- see
@@ -137,6 +137,20 @@ containing the new version number of the table after altering the columns.
***
### blobColumns()
```ts
abstract blobColumns(): Promise<string[]>
```
Blob v2 columns, including nested dotted paths.
#### Returns
`Promise`&lt;`string`[]&gt;
***
### branches()
```ts
@@ -499,6 +513,54 @@ Drop an index from the table.
***
### fetchBlobFiles()
```ts
abstract fetchBlobFiles(column, rowIds): Promise<(null | BlobFile)[]>
```
Opens lazy blob handles for `column` at the given row IDs using the
table's current checkout.
Preserves input order, duplicates, and nulls. Use this for large payloads.
See [Table.fetchBlobs](Table.md#fetchblobs) for row-ID validity across versions.
#### Parameters
* **column**: `string`
* **rowIds**: readonly (`number` \| `bigint`)[]
#### Returns
`Promise`&lt;(`null` \| [`BlobFile`](BlobFile.md))[]&gt;
***
### fetchBlobs()
```ts
abstract fetchBlobs(column, rowIds): Promise<(null | Buffer)[]>
```
Bytes for `column` at row IDs from [Query.withRowId](Query.md#withrowid).
Reads the table's current checkout. IDs from another version can fail after
compaction unless stable row ids are enabled. Results keep input order and
duplicates. Null blobs are `null`. Empty blobs are empty buffers.
#### Parameters
* **column**: `string`
* **rowIds**: readonly (`number` \| `bigint`)[]
#### Returns
`Promise`&lt;(`null` \| `Buffer`)[]&gt;
***
### flushLsm()
```ts
@@ -676,9 +738,17 @@ List all the versions of the table
abstract mergeInsert(on): MergeInsertBuilder
```
Create a [MergeInsertBuilder](MergeInsertBuilder.md), which combines new data with the
existing table in a single transaction — inserting, updating and deleting
rows depending on how they match.
#### Parameters
* **on**: `string` \| `string`[]
The column, or columns, to match source rows against target
rows on. Typically a key or id column. Several columns match on the
composite key: a source row updates a target row only when it agrees on
every one of them.
#### Returns
@@ -846,10 +916,10 @@ abstract refreshColumn(column): Promise<RefreshColumnResult>
Fill the rows of a computed column that hold no value yet.
Rows appended since the last refresh are filled by the next one; rows
already filled are left as they are, so the call is idempotent and does
not observe a mutated input. Local tables only: a remote refresh runs
as a server job, through [Table#refreshColumnAsync](Table.md#refreshcolumnasync).
Rows appended since the last refresh are filled by the next one, and
rows whose inputs changed since they were computed are recomputed;
everything else is left as it is. Local tables only: a remote refresh
runs as a server job, through [Table#refreshColumnAsync](Table.md#refreshcolumnasync).
#### Parameters
@@ -1258,7 +1328,7 @@ value is 0")
Note: if your condition is something like "some_id_column == 7" and
you are updating many rows (with different ids) then you will get
better performance with a single [`merge_insert`] call instead of
repeatedly calilng this method.
repeatedly calling this method.
##### Parameters
docs/src/js/enumerations/ClientAuthMethod.md
+48
View File
@@ -0,0 +1,48 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / ClientAuthMethod
# Enumeration: ClientAuthMethod
How the client authenticates to the OAuth token endpoint.
The method applies to every OAuth request that carries client
authentication: client-credentials, authorization-code exchange,
refresh-token, and device-authorization requests. The Azure managed
identity flow ignores this option.
## Enumeration Members
### ClientSecretBasic
```ts
ClientSecretBasic: "client_secret_basic";
```
HTTP Basic authentication. This is the RFC 6749 recommended method and
the normal default for confidential clients, including default Okta
applications. Requires `clientSecret`.
***
### ClientSecretPost
```ts
ClientSecretPost: "client_secret_post";
```
Credentials in the request body, for providers configured to require it.
Requires `clientSecret`.
***
### None
```ts
None: "none";
```
No client authentication, for public clients using PKCE or the device
flow. Cannot be combined with `clientSecret`.
docs/src/js/enumerations/OAuthFlowType.md
+20
View File
@@ -10,6 +10,16 @@ OAuth authentication flow types.
## Enumeration Members
### AuthorizationCode
```ts
AuthorizationCode: "authorization_code";
```
Interactive Authorization Code grant, using PKCE by default.
***
### AzureManagedIdentity
```ts
@@ -27,3 +37,13 @@ ClientCredentials: "client_credentials";
```
Client Credentials grant (service-to-service / M2M).
***
### DeviceCode
```ts
DeviceCode: "device_code";
```
Device Authorization grant for CLI and headless environments.
docs/src/js/functions/blob.md
+55
View File
@@ -0,0 +1,55 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / blob
# Function: blob()
```ts
function blob(name, options): Field
```
Declares a `lance.blob.v2` column.
Query results are descriptors, not payload bytes. Use [Table.fetchBlobs](../classes/Table.md#fetchblobs)
or [Table.fetchBlobFiles](../classes/Table.md#fetchblobfiles) to read bytes.
## Parameters
* **name**: `string`
* **options**: [`BlobOptions`](../type-aliases/BlobOptions.md) = `{}`
## Returns
`Field`
## Example
```ts
import { readFile } from "node:fs/promises";
import { Field, Int64, Schema } from "apache-arrow";
import { blob, connect } from "@lancedb/lancedb";
const db = await connect("./data");
const video = await readFile("clip.mp4");
const table = await db.createTable(
"videos",
[{ id: 1n, video }],
{
schema: new Schema([
new Field("id", new Int64()),
blob("video"),
]),
},
);
const rows = await table.query().select(["id"]).withRowId().toArray();
const rowIds = rows.map((row) => row._rowid as bigint);
const bytes = await table.fetchBlobs("video", rowIds);
const [handle] = await table.fetchBlobFiles("video", rowIds);
const size = handle!.size();
const header = await handle!.readRange(0n, size < 65536n ? size : 65536n);
```
docs/src/js/functions/connectCatalog.md
+32
View File
@@ -0,0 +1,32 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / connectCatalog
# Function: connectCatalog()
```ts
function connectCatalog(endpoint, options): Promise<Catalog>
```
Connect to an HTTP(S) catalog endpoint. Catalog requests omit database-selection
headers; opened database connections inherit authentication and client options.
## Parameters
* **endpoint**: `string`
* **options**: [`CatalogOptions`](../interfaces/CatalogOptions.md) = `{}`
## Returns
`Promise`&lt;[`Catalog`](../classes/Catalog.md)&gt;
## Example
```ts
const catalog = await connectCatalog("https://my-server.example", { apiKey: "secret" });
const db = await catalog.createDatabase("analytics", { existOk: true });
const page = await catalog.listDatabases({ limit: 20 });
```
docs/src/js/functions/isBlobField.md
+22
View File
@@ -0,0 +1,22 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / isBlobField
# Function: isBlobField()
```ts
function isBlobField(field): boolean
```
Checks for the `lance.blob.v2` extension marker. Does not validate the
field's storage type.
## Parameters
* **field**: `Field`&lt;`any`&gt;
## Returns
`boolean`
docs/src/js/functions/makeJsonField.md
+36
View File
@@ -0,0 +1,36 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / makeJsonField
# Function: makeJsonField()
```ts
function makeJsonField(name, nullable): Field
```
Create an Arrow field backed by LanceDB's JSON extension type.
## Parameters
* **name**: `string`
The field name.
* **nullable**: `boolean` = `true`
Whether the field accepts null values.
## Returns
`Field`
## Example
```ts
import { connect, makeJsonField } from "@lancedb/lancedb";
import { Schema } from "apache-arrow";
const schema = new Schema([makeJsonField("metadata")]);
const db = await connect("/path/to/database");
await db.createTable("items", [{ metadata: '{"source":"api"}' }], { schema });
```
docs/src/js/globals.md
+15 -1
View File
@@ -11,6 +11,7 @@
## Enumerations
- [ClientAuthMethod](enumerations/ClientAuthMethod.md)
- [FullTextQueryType](enumerations/FullTextQueryType.md)
- [OAuthFlowType](enumerations/OAuthFlowType.md)
- [Occur](enumerations/Occur.md)
@@ -19,10 +20,12 @@
## Classes
- [AutoQuery](classes/AutoQuery.md)
- [BlobFile](classes/BlobFile.md)
- [BooleanQuery](classes/BooleanQuery.md)
- [BoostQuery](classes/BoostQuery.md)
- [BranchContents](classes/BranchContents.md)
- [Branches](classes/Branches.md)
- [Catalog](classes/Catalog.md)
- [Connection](classes/Connection.md)
- [HeaderProvider](classes/HeaderProvider.md)
- [Index](classes/Index.md)
@@ -34,6 +37,7 @@
- [MultiMatchQuery](classes/MultiMatchQuery.md)
- [NativeJsHeaderProvider](classes/NativeJsHeaderProvider.md)
- [OAuthHeaderProvider](classes/OAuthHeaderProvider.md)
- [OAuthSession](classes/OAuthSession.md)
- [PermutationBuilder](classes/PermutationBuilder.md)
- [PhraseQuery](classes/PhraseQuery.md)
- [Query](classes/Query.md)
@@ -61,6 +65,7 @@
- [BranchIndexSummary](interfaces/BranchIndexSummary.md)
- [BranchRowCountSummary](interfaces/BranchRowCountSummary.md)
- [BucketStats](interfaces/BucketStats.md)
- [CatalogOptions](interfaces/CatalogOptions.md)
- [CherryPickError](interfaces/CherryPickError.md)
- [CherryPickPreview](interfaces/CherryPickPreview.md)
- [CherryPickResult](interfaces/CherryPickResult.md)
@@ -96,9 +101,10 @@
- [IvfFlatOptions](interfaces/IvfFlatOptions.md)
- [IvfPqOptions](interfaces/IvfPqOptions.md)
- [IvfRqOptions](interfaces/IvfRqOptions.md)
- [JobDescription](interfaces/JobDescription.md)
- [JobEventsOptions](interfaces/JobEventsOptions.md)
- [JobFailureInfo](interfaces/JobFailureInfo.md)
- [JobInfo](interfaces/JobInfo.md)
- [ListDatabasesResponse](interfaces/ListDatabasesResponse.md)
- [ListNamespacesOptions](interfaces/ListNamespacesOptions.md)
- [ListNamespacesResponse](interfaces/ListNamespacesResponse.md)
- [ListTablesOptions](interfaces/ListTablesOptions.md)
@@ -121,6 +127,8 @@
- [RestNamespaceConfig](interfaces/RestNamespaceConfig.md)
- [RetryConfig](interfaces/RetryConfig.md)
- [ScannableOptions](interfaces/ScannableOptions.md)
- [SessionLogout](interfaces/SessionLogout.md)
- [SessionStatus](interfaces/SessionStatus.md)
- [ShuffleOptions](interfaces/ShuffleOptions.md)
- [SplitCalculatedOptions](interfaces/SplitCalculatedOptions.md)
- [SplitHashOptions](interfaces/SplitHashOptions.md)
@@ -130,6 +138,7 @@
- [TableStatistics](interfaces/TableStatistics.md)
- [TimeoutConfig](interfaces/TimeoutConfig.md)
- [TlsConfig](interfaces/TlsConfig.md)
- [TokenCacheOptions](interfaces/TokenCacheOptions.md)
- [TokenResponse](interfaces/TokenResponse.md)
- [TokenizeOptions](interfaces/TokenizeOptions.md)
- [UpdateFieldMetadataResult](interfaces/UpdateFieldMetadataResult.md)
@@ -143,6 +152,7 @@
- [AnalyzePlanDistributedMetrics](type-aliases/AnalyzePlanDistributedMetrics.md)
- [BaseTokenizer](type-aliases/BaseTokenizer.md)
- [BlobOptions](type-aliases/BlobOptions.md)
- [Data](type-aliases/Data.md)
- [DataLike](type-aliases/DataLike.md)
- [FieldLike](type-aliases/FieldLike.md)
@@ -158,10 +168,14 @@
## Functions
- [RecordBatchIterator](functions/RecordBatchIterator.md)
- [blob](functions/blob.md)
- [connect](functions/connect.md)
- [connectCatalog](functions/connectCatalog.md)
- [connectNamespace](functions/connectNamespace.md)
- [instrumentLanceDbMetrics](functions/instrumentLanceDbMetrics.md)
- [isBlobField](functions/isBlobField.md)
- [makeArrowTable](functions/makeArrowTable.md)
- [makeJsonField](functions/makeJsonField.md)
- [packBits](functions/packBits.md)
- [permutationBuilder](functions/permutationBuilder.md)
- [tokenize](functions/tokenize.md)
docs/src/js/interfaces/CatalogOptions.md
+81
View File
@@ -0,0 +1,81 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / CatalogOptions
# Interface: CatalogOptions
Options shared by a catalog and the database connections it returns.
## Extends
- `Omit`&lt;`NativeCatalogOptions`, `"oauthConfig"`&gt;
## Properties
### apiKey?
```ts
optional apiKey: string;
```
#### Inherited from
`Omit.apiKey`
***
### clientConfig?
```ts
optional clientConfig: ClientConfig;
```
#### Inherited from
`Omit.clientConfig`
***
### headerProvider?
```ts
optional headerProvider: HeaderProvider | () => Record<string, string> | Promise<Record<string, string>>;
```
Called for each request to supply authentication headers.
***
### oauthConfig?
```ts
optional oauthConfig: OAuthConfig;
```
***
### readConsistencyInterval?
```ts
optional readConsistencyInterval: number;
```
#### Inherited from
`Omit.readConsistencyInterval`
***
### sqlHostOverride?
```ts
optional sqlHostOverride: string;
```
SQL service endpoint inherited by database connections.
#### Inherited from
`Omit.sqlHostOverride`
docs/src/js/interfaces/ClientConfig.md
+4
View File
@@ -22,6 +22,10 @@ optional extraHeaders: Record<string, string>;
optional idDelimiter: string;
```
The delimiter joining a namespace path and a name into one object
identifier. `"$"` is the only supported value, and leaving this unset is
how to get it; anything else is rejected when the connection is created.
***
### retryConfig?
docs/src/js/interfaces/HnswPqOptions.md
+1 -1
View File
@@ -118,7 +118,7 @@ Number of sub-vectors of PQ.
This value controls how much the vector is compressed during the quantization step.
The more sub vectors there are the less the vector is compressed. The default is
the dimension of the vector divided by 16. If the dimension is not evenly divisible
by 16 we use the dimension divded by 8.
by 16 we use the dimension divided by 8.
The above two cases are highly preferred. Having 8 or 16 values per subvector allows
us to use efficient SIMD instructions.
docs/src/js/interfaces/IndexOptions.md
+1 -1
View File
@@ -16,7 +16,7 @@ optional config: Index;
Advanced index configuration
This option allows you to specify a specfic index to create and also
This option allows you to specify a specific index to create and also
allows you to pass in configuration for training the index.
See the static methods on Index for details on the various index types.
docs/src/js/interfaces/IvfPqOptions.md
+1 -1
View File
@@ -112,7 +112,7 @@ Number of sub-vectors of PQ.
This value controls how much the vector is compressed during the quantization step.
The more sub vectors there are the less the vector is compressed. The default is
the dimension of the vector divided by 16. If the dimension is not evenly divisible
by 16 we use the dimension divded by 8.
by 16 we use the dimension divided by 8.
The above two cases are highly preferred. Having 8 or 16 values per subvector allows
us to use efficient SIMD instructions.
docs/src/js/interfaces/JobDescription.md
-66
View File
@@ -1,66 +0,0 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / JobDescription
# Interface: JobDescription
A described job from `Connection.getJob`.
## Properties
### creationMs
```ts
creationMs: number;
```
When the job was created, in milliseconds since the epoch.
***
### failure?
```ts
optional failure: JobFailureInfo;
```
Why the job failed, when the job is failed and the server reports a
reason.
***
### jobId
```ts
jobId: string;
```
***
### jobType
```ts
jobType: string;
```
***
### specJson?
```ts
optional specJson: string;
```
The job-type-specific specification as a JSON string, when present.
***
### state
```ts
state: string;
```
Lifecycle state: "running", "finished", "failed", or "cancelled".
docs/src/js/interfaces/JobEventsOptions.md
+29
View File
@@ -0,0 +1,29 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / JobEventsOptions
# Interface: JobEventsOptions
Which of a job's events [Job.events](../classes/Job.md#events) returns.
## Properties
### filter?
```ts
optional filter: string;
```
SQL-like filter over the event columns.
***
### limit?
```ts
optional limit: number;
```
Maximum event rows to return, up to the server maximum of 10,000.
docs/src/js/interfaces/JobInfo.md
+1 -1
View File
@@ -26,7 +26,7 @@ When the job was created, in milliseconds since the epoch.
jobId: string;
```
The job id -- what `Connection.getJob` and `Connection.cancelJob`
The job id -- what `Connection.openJob` and `Connection.cancelJob`
accept.
***
docs/src/js/interfaces/ListDatabasesResponse.md
+23
View File
@@ -0,0 +1,23 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / ListDatabasesResponse
# Interface: ListDatabasesResponse
## Properties
### databases
```ts
databases: string[];
```
***
### pageToken?
```ts
optional pageToken: string;
```
docs/src/js/interfaces/MaterializedViewDefinition.md
+10
View File
@@ -50,6 +50,16 @@ projections: [string, string][];
***
### sourceNamespace
```ts
sourceNamespace: string[];
```
Namespace holding the source table; empty is the root namespace.
***
### sourceTable
```ts
docs/src/js/interfaces/NativeOAuthConfig.md
+76 -1
View File
@@ -15,6 +15,39 @@ All token acquisition and refresh is handled in the Rust layer.
## Properties
### audience?
```ts
optional audience: string;
```
Optional provider-specific audience for authorization and token requests.
***
### callbackPort?
```ts
optional callbackPort: number;
```
Port for the authorization_code loopback callback server.
***
### clientAuthMethod?
```ts
optional clientAuthMethod: string;
```
How the client authenticates to the token endpoint: "none",
"client_secret_basic", or "client_secret_post". Defaults to
"client_secret_basic" when a client secret is set, and "none" for
public clients.
***
### clientId
```ts
@@ -41,7 +74,8 @@ Client secret (required for client_credentials).
optional flow: string;
```
Authentication flow: "client_credentials" or "azure_managed_identity"
Authentication flow: "client_credentials", "authorization_code",
"device_code", or "azure_managed_identity"
***
@@ -66,6 +100,16 @@ Client ID for user-assigned managed identity (azure_managed_identity).
***
### redirectUri?
```ts
optional redirectUri: string;
```
Loopback redirect URI for authorization_code.
***
### refreshBufferSecs?
```ts
@@ -78,6 +122,16 @@ the TTL, each request refreshes the token.
***
### resource?
```ts
optional resource: string;
```
Optional resource indicator for authorization and token requests.
***
### scopes
```ts
@@ -86,3 +140,24 @@ scopes: string[];
OAuth scopes to request. For Azure managed identity, exactly one scope
or resource is required. For example: `["api://{app_id}/.default"]`
***
### tokenCache?
```ts
optional tokenCache: TokenCacheOptions;
```
Opt in to the persistent token cache so short-lived processes reuse
one session. Only refresh tokens are persisted.
***
### usePkce?
```ts
optional usePkce: boolean;
```
Whether authorization_code uses S256 PKCE (default: true).
docs/src/js/interfaces/OAuthConfig.md
+106
View File
@@ -26,6 +26,18 @@ const config: OAuthConfig = {
};
```
Providers requiring an explicit target can set `resource` and/or `audience`:
```typescript
const targeted: OAuthConfig = {
issuerUrl: "https://issuer.example.com",
clientId: "app-id",
clientSecret: "secret",
scopes: ["read"],
resource: "https://api.example.com",
audience: "lancedb-api",
};
```
```typescript
const config: OAuthConfig = {
issuerUrl: "https://login.microsoftonline.com/{tenant}/v2.0",
@@ -35,8 +47,57 @@ const config: OAuthConfig = {
};
```
The authorization URL is written to stderr before LanceDB tries to open a
browser, so it can be copied in headless environments.
```typescript
const config: OAuthConfig = {
issuerUrl: "https://login.microsoftonline.com/{tenant}/v2.0",
clientId: "app-id",
scopes: ["openid", "api://lancedb-api/access"],
flow: OAuthFlowType.AuthorizationCode,
};
```
Device Authorization writes the verification URL and user code to stderr
before polling begins.
## Properties
### audience?
```ts
optional audience: string;
```
Provider-specific audience, forwarded to authorization and token endpoints,
including refresh requests. Not supported for Azure managed identity.
***
### callbackPort?
```ts
optional callbackPort: number;
```
Port for the AuthorizationCode loopback callback server (default: 8400).
***
### clientAuthMethod?
```ts
optional clientAuthMethod: ClientAuthMethod;
```
How the client authenticates to the token endpoint (default: auto).
With a `clientSecret` the default is `ClientAuthMethod.ClientSecretBasic`,
which matches the RFC 6749 recommendation and the default configuration
of Okta confidential applications; without a secret the client is public
and no client authentication is sent.
***
### clientId
```ts
@@ -88,6 +149,16 @@ Client ID for user-assigned managed identity (AzureManagedIdentity).
***
### redirectUri?
```ts
optional redirectUri: string;
```
Loopback redirect URI for AuthorizationCode.
***
### refreshBufferSecs?
```ts
@@ -100,6 +171,18 @@ the TTL, each request refreshes the token.
***
### resource?
```ts
optional resource: string;
```
Resource indicator (RFC 8707), forwarded verbatim to authorization and token
endpoints, including refresh requests. Must be an absolute URI without a
fragment. Not supported for Azure managed identity.
***
### scopes
```ts
@@ -109,3 +192,26 @@ scopes: string[];
OAuth scopes to request.
For Azure managed identity, exactly one scope or resource is required.
For example: `["api://{app_id}/.default"]`
***
### tokenCache?
```ts
optional tokenCache: TokenCacheOptions;
```
Opt in to the persistent token cache so short-lived processes reuse one
session. Only refresh tokens are persisted. Only supported by
AuthorizationCode and DeviceCode; Azure managed identity is rejected.
Default: unset (memory only).
***
### usePkce?
```ts
optional usePkce: boolean;
```
Protect AuthorizationCode with S256 PKCE (default: true).
docs/src/js/interfaces/OptimizeOptions.md
+2 -1
View File
@@ -26,7 +26,8 @@ const olderThan = new Date();
olderThan.setDate(olderThan.getDate() - 1));
tbl.optimize({cleanupOlderThan: olderThan});
// Delete all versions except the current version
// Delete versions committed before this point. Versions created by the
// optimize call itself are newer than the cutoff and will be retained.
tbl.optimize({cleanupOlderThan: new Date()});
```
docs/src/js/interfaces/SessionLogout.md
+20
View File
@@ -0,0 +1,20 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / SessionLogout
# Interface: SessionLogout
Result of [OAuthSession.logout](../classes/OAuthSession.md#logout).
## Properties
### removed
```ts
removed: boolean;
```
Whether a cached credential was removed. `false` means no matching
session was cached; logout is idempotent.
docs/src/js/interfaces/SessionStatus.md
+94
View File
@@ -0,0 +1,94 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / SessionStatus
# Interface: SessionStatus
Safe, non-secret view of a cached OAuth session, returned by
[OAuthSession.status](../classes/OAuthSession.md#status) and [OAuthSession.login](../classes/OAuthSession.md#login).
## Properties
### audience?
```ts
optional audience: string;
```
Provider-specific audience used to obtain the cached session, if configured.
***
### clientId
```ts
clientId: string;
```
Client ID of the cached session.
***
### flow
```ts
flow: string;
```
Flow that produced the cached session.
***
### issuerUrl
```ts
issuerUrl: string;
```
Canonical issuer URL of the cached session.
***
### obtainedAt?
```ts
optional obtainedAt: number;
```
When the cached session was obtained, as Unix seconds.
***
### refreshable
```ts
refreshable: boolean;
```
Whether a cached session exists that can obtain tokens without
interactive authentication. Because access tokens are not persisted,
this is `true` exactly when a refresh token is cached; the next
connection refreshes with it rather than opening a browser or device
prompt.
***
### resource?
```ts
optional resource: string;
```
Resource indicator used to obtain the cached session, if configured.
***
### scopes
```ts
scopes: string[];
```
Canonical (sorted, de-duplicated) scope set of the cached session.
docs/src/js/interfaces/TokenCacheOptions.md
+41
View File
@@ -0,0 +1,41 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / TokenCacheOptions
# Interface: TokenCacheOptions
Options for the persistent OAuth token cache.
The cache is opt-in: it is only used when set as `tokenCache` on
[OAuthConfig](OAuthConfig.md). Only refresh tokens are persisted, in a private
directory with owner-only permissions, so short-lived processes can reuse
an authenticated session instead of re-prompting on every start.
Multiple identities (issuer, client, scopes, resource, audience, flow, client authentication)
get separate cache entries. Within one identity the most recent login wins.
## Properties
### cacheDir?
```ts
optional cacheDir: string;
```
Directory that holds cached credentials. Defaults to
`$XDG_CACHE_HOME/lancedb/oauth`, `$HOME/.cache/lancedb/oauth` on Unix,
or `%LOCALAPPDATA%\\lancedb\\oauth` on Windows. The directory is created
with owner-only permissions (`0700`) when missing.
***
### lockTimeoutSecs?
```ts
optional lockTimeoutSecs: number;
```
How long to wait for the cross-process refresh lock before failing, in
seconds (default: 30).
docs/src/js/type-aliases/BlobOptions.md
+48
View File
@@ -0,0 +1,48 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobOptions
# Type Alias: BlobOptions
```ts
type BlobOptions: object;
```
## Type declaration
### dedicatedSizeThreshold?
```ts
optional dedicatedSizeThreshold: number;
```
Max payload bytes stored in a packed sidecar before a dedicated file. Must
be a positive safe integer.
### inlineSizeThreshold?
```ts
optional inlineSizeThreshold: number;
```
Max payload bytes kept inline in the data file. Zero is allowed. Must be a
safe integer.
### nullable?
```ts
optional nullable: boolean;
```
Defaults to true.
### packFileSizeThreshold?
```ts
optional packFileSizeThreshold: number;
```
Max bytes in one packed sidecar before starting another. Must be a positive
safe integer.
docs/src/python/python.md
+94
View File
@@ -28,6 +28,70 @@ is also an [asynchronous API client](#connections-asynchronous).
::: lancedb.Session
## Catalogs (Synchronous)
Remote catalogs manage databases through a server's root namespace. Opened databases
are ordinary connections. Dropping a database requires it to be empty.
::: lancedb.connect_catalog
::: lancedb.catalog.Catalog
::: lancedb.catalog.ListDatabasesResponse
## Remote SQL
Submit SQL against a remote LanceDB database through the connection.
The connected database and `default_namespace_path=["public"]` are used for
unqualified tables. Fully qualified references can still query other databases
and namespaces available to the same deployment. `execute_query` returns a
reader as soon as its initial result stream is available. `execute_query_async`
returns a query handle immediately; use it to inspect progress, open a reader,
or cancel the query. The SQL client is initialized by the first query and
retained for the lifetime of the remote connection. Query ids are random,
connection-scoped references rather than encoded SQL or durable resume tokens:
```python
import lancedb
db = lancedb.connect(
"db://analytics",
api_key="ldb_...",
host_override="https://api.example.com",
sql_host_override="grpc+tls://sql.example.com:10026",
)
reader = db.execute_query(
"""
SELECT events.id, accounts.name
FROM analytics.public.events AS events
JOIN users.public.accounts AS accounts ON events.user_id = accounts.id
""",
default_namespace_path=["public"],
)
for batch in reader:
print(batch.num_rows)
query = db.execute_query_async("SELECT * FROM events")
print(query.id)
print(query.describe().status)
for batch in query.reader():
print(batch.num_rows)
# The async connection exposes the same lifecycle without blocking:
# async_db = await lancedb.connect_async(
# "db://analytics",
# api_key="ldb_...",
# host_override="https://api.example.com",
# sql_host_override="grpc+tls://sql.example.com:10026",
# )
# reader = await async_db.execute_query("SELECT * FROM events")
# query = await async_db.execute_query_async("SELECT * FROM events")
# description = await async_db.describe_query(query.id)
# async for batch in await query.reader():
# print(batch.num_rows)
# await query.cancel()
```
## Namespaces (Synchronous)
A namespace-backed connection resolves tables through a
@@ -74,6 +138,10 @@ listing a storage directory.
::: lancedb.functions.UdfDefinition
::: lancedb.secrets.EnvVarSecret
::: lancedb.secrets.SecretInfo
::: lancedb.functions.FunctionRegistrationRequest
::: lancedb.functions.FunctionArtifactRequest
@@ -82,6 +150,8 @@ listing a storage directory.
::: lancedb.functions.PythonAdapterSpec
::: lancedb.functions.FunctionImage
::: lancedb.functions.FunctionVersion
::: lancedb.functions.PythonRuntimeSpec
@@ -96,6 +166,8 @@ listing a storage directory.
::: lancedb.functions.OutputMapping
::: lancedb.functions.AssignmentMapping
::: lancedb.functions.FunctionBinding
::: lancedb.functions.RefreshColumnResult
@@ -104,6 +176,18 @@ listing a storage directory.
::: lancedb.job.AsyncJob
::: lancedb.job.JobInfo
::: lancedb.job.JobDescription
::: lancedb.job.JobFailureInfo
::: lancedb.sql.Query
::: lancedb.sql.AsyncQuery
::: lancedb.sql.QueryDescription
## Materialized Views (Synchronous)
::: lancedb.materialized_view.MaterializedView
@@ -251,6 +335,12 @@ still work. Queries return descriptors. Call
::: lancedb.exceptions.MissingColumnError
::: lancedb.exceptions.JobNotFoundError
::: lancedb.exceptions.JobFailedError
::: lancedb.exceptions.JobCancelledError
## Integrations
## Pydantic
@@ -288,6 +378,10 @@ still work. Queries return descriptors. Call
## Connections (Asynchronous)
::: lancedb.connect_catalog_async
::: lancedb.catalog.AsyncCatalog
Connections represent a connection to a LanceDb database and
can be used to create, list, or open tables.
docs/tsconfig.json
-17
View File
@@ -1,17 +0,0 @@
{
"include": [
"src/*.ts",
],
"compilerOptions": {
"target": "es2022",
"module": "nodenext",
"declaration": true,
"outDir": "./dist",
"strict": true,
"allowJs": true,
"resolveJsonModule": true,
},
"exclude": [
"./dist/*",
]
}
java/lancedb-core/pom.xml
+1 -1
View File
@@ -8,7 +8,7 @@
<parent>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.38.0-beta.12</version>
<version>0.40.0-beta.1</version>
<relativePath>../pom.xml</relativePath>
</parent>
java/pom.xml
+2 -2
View File
@@ -6,7 +6,7 @@
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.38.0-beta.12</version>
<version>0.40.0-beta.1</version>
<packaging>pom</packaging>
<name>${project.artifactId}</name>
<description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<arrow.version>15.0.0</arrow.version>
<lance-core.version>12.0.0-beta.2</lance-core.version>
<lance-core.version>13.0.0-beta.4</lance-core.version>
<spotless.skip>false</spotless.skip>
<spotless.version>2.30.0</spotless.version>
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
nodejs/Cargo.toml
+4 -3
View File
@@ -1,7 +1,7 @@
[package]
name = "lancedb-nodejs"
edition.workspace = true
version = "0.38.0-beta.12"
version = "0.40.0-beta.1"
publish = false
license.workspace = true
description.workspace = true
@@ -37,8 +37,9 @@ lzma-sys = { version = "0.1", features = ["static"] }
log.workspace = true
# Pin to resolve build failures; update periodically for security patches.
aws-lc-sys = "=0.40.0"
aws-lc-rs = "=1.16.3"
# rustls >= 0.23.45 (RUSTSEC-2026-0285) needs aws-lc-rs >= 1.18.
aws-lc-sys = "=0.45.0"
aws-lc-rs = "=1.18.1"
[build-dependencies]
napi-build = "2.3.1"
nodejs/__test__/arrow.test.ts
+16
View File
@@ -20,6 +20,7 @@ import {
fromTableToBuffer,
makeArrowTable,
makeEmptyTable,
makeJsonField,
} from "../lancedb/arrow";
import {
EmbeddingFunction,
@@ -28,6 +29,21 @@ import {
import { EmbeddingFunctionConfig } from "../lancedb/embedding/registry";
import { sanitizeTable } from "../lancedb/sanitize";
it("creates a nullable JSON field with the Arrow extension metadata", () => {
const field = makeJsonField("metadata");
expect(field.name).toBe("metadata");
expect(field.type).toEqual(new arrow15.Utf8());
expect(field.nullable).toBe(true);
expect(field.metadata).toEqual(
new Map([["ARROW:extension:name", "arrow.json"]]),
);
});
it("allows JSON fields to be non-nullable", () => {
expect(makeJsonField("metadata", false).nullable).toBe(false);
});
// biome-ignore lint/suspicious/noExplicitAny: skip
function sampleRecords(): Array<Record<string, any>> {
return [
nodejs/__test__/blob.test.ts
+185
View File
@@ -0,0 +1,185 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { Field, Int64, List, Schema, Struct, Utf8 } from "apache-arrow";
import { makeArrowTable } from "../lancedb/arrow";
import { BlobFile, blob, coerceBlobValue, isBlobField } from "../lancedb/blob";
describe("blob()", () => {
it("marks the field as lance.blob.v2", () => {
const field = blob("image", { nullable: false });
expect(field.nullable).toBe(false);
expect(isBlobField(field)).toBe(true);
expect(field.metadata.get("ARROW:extension:name")).toBe("lance.blob.v2");
});
it("writes encoding thresholds as field metadata", () => {
const field = blob("video", {
inlineSizeThreshold: 1024,
dedicatedSizeThreshold: 2 * 1024 * 1024,
packFileSizeThreshold: 64 * 1024 * 1024,
});
expect(
field.metadata.get("lance-encoding:blob-inline-size-threshold"),
).toBe("1024");
expect(
field.metadata.get("lance-encoding:blob-dedicated-size-threshold"),
).toBe(String(2 * 1024 * 1024));
expect(
field.metadata.get("lance-encoding:blob-pack-file-size-threshold"),
).toBe(String(64 * 1024 * 1024));
});
it("rejects invalid thresholds", () => {
expect(() => blob("image", { inlineSizeThreshold: -1 })).toThrow(
/inlineSizeThreshold must be non-negative/,
);
expect(() => blob("image", { dedicatedSizeThreshold: 0 })).toThrow(
/dedicatedSizeThreshold must be positive/,
);
expect(() => blob("image", { packFileSizeThreshold: 1.5 })).toThrow(
/packFileSizeThreshold must be a safe integer/,
);
expect(() =>
blob("image", { dedicatedSizeThreshold: Number.MAX_SAFE_INTEGER + 1 }),
).toThrow(/dedicatedSizeThreshold must be a safe integer/);
});
});
describe("coerceBlobValue", () => {
it.each([
["Buffer", Buffer.from("x"), { data: Buffer.from("x"), uri: null }],
[
"Uint8Array",
new Uint8Array([120]),
{ data: new Uint8Array([120]), uri: null },
],
["URI string", "s3://bucket/key", { data: null, uri: "s3://bucket/key" }],
[
"data struct",
{ data: Buffer.from("y") },
{ data: Buffer.from("y"), uri: null },
],
[
"uri struct",
{ uri: "s3://bucket/key" },
{ data: null, uri: "s3://bucket/key" },
],
["null", null, null],
])("accepts %s", (_name, input, expected) => {
expect(coerceBlobValue(input)).toEqual(expected);
});
it.each([
["empty URI", "", /uri cannot be empty/],
["object without data or uri", { position: 0 }, /data' or 'uri/],
[
"Int16Array",
new Int16Array([1]),
/Blob data must be Buffer or Uint8Array/,
],
[
"both data and uri",
{ data: Buffer.from("y"), uri: "s3://bucket/key" },
/exactly one of 'data' or 'uri'/,
],
[
"neither data nor uri",
{ data: null, uri: null },
/exactly one of 'data' or 'uri'/,
],
])("rejects %s", (_name, input, message) => {
expect(() => coerceBlobValue(input)).toThrow(message);
});
});
describe("BlobFile", () => {
it("rejects constructing BlobFile without a native handle", () => {
expect(() => new (BlobFile as unknown as { new (): BlobFile })()).toThrow(
/fetchBlobFiles/,
);
});
});
describe("makeArrowTable blob columns", () => {
it("coerces Buffer input onto a blob field", () => {
const schema = new Schema([
new Field("id", new Int64(), true),
blob("image"),
]);
const table = makeArrowTable([{ id: 1n, image: Buffer.from("hello") }], {
schema,
});
expect(isBlobField(table.schema.fields[1])).toBe(true);
const image = table.getChild("image")!;
expect(image.nullCount).toBe(0);
expect(image.getChild("uri")!.get(0)).toBeNull();
expect(image.getChild("data")!.nullCount).toBe(0);
expect(Buffer.from(image.getChild("data")!.get(0)!).toString()).toBe(
"hello",
);
});
it("coerces Buffer elements inside a list and keeps null slots", () => {
const schema = new Schema([
new Field("id", new Int64(), true),
new Field("images", new List(blob("image")), true),
]);
const table = makeArrowTable(
[
{ id: 1n, images: [Buffer.from("a"), Buffer.from("bb")] },
{ id: 2n, images: null },
{ id: 3n, images: [Buffer.from("c"), null] },
{ id: 4n, images: [] },
],
{ schema },
);
const images = table.getChild("images")!;
expect(images.nullCount).toBe(1);
const rows = images.toArray();
expect(rows[1]).toBeNull();
expect(Array.from(rows[3] as Iterable<unknown>)).toHaveLength(0);
const first = Array.from(rows[0] as Iterable<{ data: Uint8Array | null }>);
expect(Buffer.from(first[0].data!).toString()).toBe("a");
expect(Buffer.from(first[1].data!).toString()).toBe("bb");
const third = Array.from(
rows[2] as Iterable<{ data: Uint8Array | null } | null>,
);
expect(Buffer.from(third[0]!.data!).toString()).toBe("c");
expect(third[1]).toBeNull();
});
it("coerces Buffer fields inside list structs", () => {
const schema = new Schema([
new Field("id", new Int64(), true),
new Field(
"items",
new List(
new Field(
"item",
new Struct([new Field("name", new Utf8(), true), blob("image")]),
true,
),
),
true,
),
]);
const table = makeArrowTable(
[
{
id: 1n,
items: [{ name: "one", image: Buffer.from("alpha") }],
},
],
{ schema },
);
const items = Array.from(
table.getChild("items")!.toArray()[0] as Iterable<{
name: string;
image: { data: Uint8Array | null };
}>,
);
expect(items[0].name).toBe("one");
expect(Buffer.from(items[0].image.data!).toString()).toBe("alpha");
});
});
nodejs/__test__/catalog.test.ts
+142
View File
@@ -0,0 +1,142 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import * as http from "http";
import { Catalog, connectCatalog } from "../lancedb";
type RecordedRequest = {
url: string;
headers: http.IncomingHttpHeaders;
body: Record<string, unknown>;
};
async function withCatalog(
responses: [number, unknown][],
callback: (catalog: Catalog, requests: RecordedRequest[]) => Promise<void>,
) {
const requests: RecordedRequest[] = [];
const server = http.createServer(async (req, res) => {
const chunks: Buffer[] = [];
for await (const chunk of req) chunks.push(Buffer.from(chunk));
const body = Buffer.concat(chunks).toString();
requests.push({
url: req.url ?? "",
headers: req.headers,
body: body ? JSON.parse(body) : {},
});
const [status, response] = responses.shift() ?? [
500,
{ error: "Unexpected request" },
];
res.writeHead(status, { "content-type": "application/json" });
res.end(status === 204 ? undefined : JSON.stringify(response));
});
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
const address = server.address();
if (!address || typeof address === "string")
throw new Error("Missing server address");
try {
const catalog = await connectCatalog(`http://127.0.0.1:${address.port}`, {
apiKey: "secret",
sqlHostOverride: "grpc+tls://sql.example.com:10026",
clientConfig: {
extraHeaders: {
"X-LanceDB-Database": "wrong-static",
"X-LanceDB-Database-Prefix": "wrong",
},
},
headerProvider: () => ({
"X-LanceDB-Database": "wrong-dynamic",
"X-LanceDB-Database-Prefix": "wrong",
authorization: "Bearer refreshed",
}),
});
await callback(catalog, requests);
expect(responses).toHaveLength(0);
} finally {
server.closeAllConnections();
await new Promise<void>((resolve) => server.close(() => resolve()));
}
}
describe("remote catalog", () => {
it("uses root namespace routes and preserves independent database scope", async () => {
await withCatalog(
[
[204, null],
[200, { tables: [] }],
[200, {}],
[200, { tables: [] }],
[200, { tables: [] }],
// biome-ignore lint/style/useNamingConvention: server wire format
[200, { namespaces: ["team/search"], page_token: "next" }],
[204, null],
],
async (catalog, requests) => {
const first = await catalog.createDatabase("team/search", {
existOk: true,
});
expect(await first.tableNames()).toEqual([]);
const second = await catalog.connectDatabase("other");
expect(await second.tableNames()).toEqual([]);
expect(await first.tableNames()).toEqual([]);
expect(
await catalog.listDatabases({ limit: 1, pageToken: "a/b" }),
).toEqual({ databases: ["team/search"], pageToken: "next" });
await catalog.dropDatabase("team/search", { ignoreMissing: true });
expect(requests[0].url).toBe("/v1/namespace/team%2Fsearch/create");
expect(requests[0].body).toEqual({ mode: "ExistOk" });
expect(requests[5].url).toBe(
"/v1/namespace/$/list?limit=1&page_token=a%2Fb",
);
expect(requests[6].body).toEqual({
mode: "Skip",
behavior: "Restrict",
});
for (const [i, request] of requests.entries()) {
expect(request.headers["x-lancedb-database"]).toBe(
i === 1 || i === 4 ? "team/search" : i === 3 ? "other" : undefined,
);
expect(request.headers["x-lancedb-database-prefix"]).toBeUndefined();
expect(request.headers.authorization).toBe("Bearer refreshed");
}
},
);
});
it("propagates lifecycle errors and sends restricted drops", async () => {
await withCatalog(
[
[404, {}],
[409, {}],
[400, {}],
[404, {}],
],
async (catalog, requests) => {
await expect(catalog.connectDatabase("missing")).rejects.toThrow(
"missing",
);
await expect(catalog.createDatabase("exists")).rejects.toThrow(
"exists",
);
await expect(catalog.dropDatabase("full")).rejects.toThrow();
await catalog.dropDatabase("missing", { ignoreMissing: true });
expect(requests[2].body).toEqual({
mode: "Fail",
behavior: "Restrict",
});
},
);
});
it("validates endpoints and pagination", async () => {
await expect(connectCatalog("/tmp/catalog")).rejects.toThrow();
const catalog = await connectCatalog("http://127.0.0.1:1");
for (const limit of [0, -1, 1.5, 2147483648]) {
await expect(catalog.listDatabases({ limit })).rejects.toThrow("limit");
}
await expect(catalog.connectDatabase("a$b")).rejects.toThrow(
"Invalid database name",
);
});
});
nodejs/__test__/fixtures/oauth_browser.cmd
+3
View File
@@ -0,0 +1,3 @@
@rem SPDX-License-Identifier: Apache-2.0
@rem SPDX-FileCopyrightText: Copyright The LanceDB Authors
@exit /b 0
nodejs/__test__/materialized_view.test.ts
+42 -6
View File
@@ -48,17 +48,35 @@ describe("materialized views", () => {
expect(definitionFromMetadata(safe, "v").limit).toBe(42);
});
it("reads the namespaced select kind and refuses unknown kinds", () => {
// "namespaced_select" is the namespaced form of "select": same shape, a
// separate kind so readers that predate it refuse instead of resolving
// the source at the root.
const namespaced = new Map([
[
DEFINITION_META_KEY,
'{"kind":"namespaced_select","source_table":"people","source_namespace":["ns"]}',
],
]);
const definition = definitionFromMetadata(namespaced, "v");
expect(definition.sourceTable).toBe("people");
expect(definition.sourceNamespace).toEqual(["ns"]);
const unknown = new Map([
[DEFINITION_META_KEY, '{"kind":"select_v3","source_table":"people"}'],
]);
expect(() => definitionFromMetadata(unknown, "v")).toThrow(
/cannot refresh/,
);
});
it("creates, refreshes and queries a view", async () => {
const view = await db.createMaterializedView("adults", "people", {
select: ["name", ["shout", "upper(name)"]],
where: "age >= 18",
});
expect(view.name).toBe("adults");
expect(await view.table().countRows()).toBe(0);
const result = await view.refresh();
expect(result.mode).toBe("rebuild");
expect(Number(result.rowsWritten)).toBe(2);
expect(await view.table().countRows()).toBe(2);
const rows = await view.table().query().toArray();
expect(rows.map((r) => r.shout).sort()).toEqual(["ADA", "GRACE"]);
@@ -80,7 +98,9 @@ describe("materialized views", () => {
});
it("refreshes incrementally after an append", async () => {
const view = await db.createMaterializedView("copy", "people");
const view = await db.createMaterializedView("copy", "people", {
withNoData: true,
});
await view.refresh();
const people = await db.openTable("people");
@@ -101,6 +121,21 @@ describe("materialized views", () => {
await expect(db.openMaterializedView("people")).rejects.toThrow(
"not a materialized view",
);
await expect(db.dropMaterializedView("people")).rejects.toThrow(
"not a materialized view",
);
await db.dropMaterializedView("adults");
expect(await db.listMaterializedViews()).toEqual([]);
});
it("returns a job when dropping a view asynchronously", async () => {
await db.createMaterializedView("adults", "people");
const job = await db.dropMaterializedViewAsync("adults");
expect(job.id).toBeNull();
await job.wait();
expect(await db.listMaterializedViews()).toEqual([]);
});
it("rejects an invalid expression at create time", async () => {
@@ -133,6 +168,7 @@ describe("materialized views", () => {
});
const view = await db.createMaterializedView("quoted", "odd_names", {
select: ["order item"],
withNoData: true,
});
const result = await view.refresh();
expect(Number(result.rowsWritten)).toBe(1);
nodejs/__test__/oauth.test.ts
+245
View File
@@ -0,0 +1,245 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import * as fs from "fs";
import * as http from "http";
import { execFileSync } from "node:child_process";
import * as os from "os";
import * as path from "path";
import { OAuthConfig, OAuthFlowType, OAuthSession } from "../lancedb/oauth";
function tempCacheDir(): string {
return fs.mkdtempSync(path.join(os.tmpdir(), "lancedb-oauth-cache-"));
}
function deviceConfig(issuerUrl: string, cacheDir: string): OAuthConfig {
return {
issuerUrl,
clientId: "client-id",
scopes: ["openid"],
flow: OAuthFlowType.DeviceCode,
tokenCache: { cacheDir },
};
}
describe("OAuthSession", () => {
beforeAll(() => {
// Child processes inherit the real environment, just as Rust reads it.
// Fail before login if the browser override only exists in Jest's sandbox.
const browser = execFileSync(
process.execPath,
["-p", "process.env.LANCEDB_OAUTH_BROWSER ?? ''"],
{ encoding: "utf8" },
).trim();
expect(browser).not.toBe("");
expect(browser).toBe(process.env.LANCEDB_OAUTH_BROWSER);
});
it("reports an absent session and logout is idempotent", async () => {
const cacheDir = tempCacheDir();
const session = new OAuthSession(
deviceConfig("https://issuer.example.com", cacheDir),
);
const status = await session.status();
expect(status.refreshable).toBe(false);
expect(status.issuerUrl).toBe("https://issuer.example.com");
expect(status.clientId).toBe("client-id");
expect(status.scopes).toEqual(["openid"]);
expect(status.flow).toBe("device_code");
expect(status.obtainedAt).toBeUndefined();
const logout = await session.logout();
expect(logout.removed).toBe(false);
});
it("requires token cache options", () => {
const config: OAuthConfig = {
issuerUrl: "https://issuer.example.com",
clientId: "client-id",
scopes: ["openid"],
flow: OAuthFlowType.DeviceCode,
};
expect(() => new OAuthSession(config)).toThrow(/token/);
});
it("rejects azure managed identity persistence", () => {
const config: OAuthConfig = {
issuerUrl: "https://login.microsoftonline.com/tenant/v2.0",
clientId: "app-id",
scopes: ["api://app/.default"],
flow: OAuthFlowType.AzureManagedIdentity,
tokenCache: { cacheDir: tempCacheDir() },
};
expect(() => new OAuthSession(config)).toThrow(/AzureManagedIdentity/);
});
it.each([
{},
{
resource: "https://api.example.com/a?x=1&y=two",
audience: "audience + & / ü",
},
])(
"logs in via device flow with target %j, caches, and logs out",
async (target) => {
const server = new MockIdp();
await server.start();
try {
const cacheDir = tempCacheDir();
const issuerUrl = server.issuerUrl();
const config = { ...deviceConfig(issuerUrl, cacheDir), ...target };
const session = new OAuthSession(config);
const status = await session.login();
expect(status.refreshable).toBe(true);
expect(status.resource).toBe(config.resource);
expect(status.audience).toBe(config.audience);
expect(status.obtainedAt).toBeGreaterThan(0);
expect(server.state.deviceAuthorizations).toBe(1);
// An independent session (a fresh "process") sees the cached login.
const other = new OAuthSession(config);
const cached = await other.status();
expect(cached.refreshable).toBe(true);
const logout = await other.logout();
expect(logout.removed).toBe(true);
const again = await session.logout();
expect(again.removed).toBe(false);
expect((await session.status()).refreshable).toBe(false);
// Only the initial login used the interactive device flow.
expect(server.state.deviceAuthorizations).toBe(1);
expect(server.state.refreshGrants).toBe(0);
expect(server.requests).toHaveLength(2);
for (const params of server.requests) {
expect(params.getAll("resource")).toEqual(
config.resource === undefined ? [] : [config.resource],
);
expect(params.getAll("audience")).toEqual(
config.audience === undefined ? [] : [config.audience],
);
}
} finally {
server.close();
}
},
15000,
);
});
/** Mock IdP with discovery, device authorization, and rotating refresh. */
class MockIdp {
readonly requests: URLSearchParams[] = [];
readonly state = {
deviceAuthorizations: 0,
refreshGrants: 0,
accessTokensIssued: 0,
currentRefresh: null as string | null,
};
private server?: http.Server;
private port = 0;
issuerUrl(): string {
return `http://127.0.0.1:${this.port}`;
}
async start(): Promise<void> {
const server = http.createServer((req, res) => {
const chunks: Buffer[] = [];
req.on("data", (chunk) => chunks.push(chunk));
req.on("end", () => {
const body = Buffer.concat(chunks).toString();
const params = new URLSearchParams(body);
this.handle(req.url ?? "", params, res);
});
});
await new Promise<void>((resolve) => {
server.listen(0, "127.0.0.1", () => resolve());
});
const address = server.address();
if (address && typeof address === "object") {
this.port = address.port;
}
this.server = server;
}
private handle(
url: string,
params: URLSearchParams,
res: http.ServerResponse,
): void {
const respond = (status: number, payload: unknown): void => {
const body = JSON.stringify(payload);
res.writeHead(status, {
"Content-Type": "application/json",
"Content-Length": Buffer.byteLength(body),
});
res.end(body);
};
if (url === "/.well-known/openid-configuration") {
respond(200, {
// biome-ignore lint/style/useNamingConvention: OAuth wire format
token_endpoint: `${this.issuerUrl()}/token`,
// biome-ignore lint/style/useNamingConvention: OAuth wire format
device_authorization_endpoint: `${this.issuerUrl()}/device`,
});
return;
}
if (url === "/device" || url === "/token") {
this.requests.push(params);
}
if (url === "/device") {
this.state.deviceAuthorizations += 1;
respond(200, {
// biome-ignore lint/style/useNamingConvention: OAuth wire format
device_code: "device-code",
// biome-ignore lint/style/useNamingConvention: OAuth wire format
user_code: "ABCD-EFGH",
// biome-ignore lint/style/useNamingConvention: OAuth wire format
verification_uri: `${this.issuerUrl()}/verify`,
// biome-ignore lint/style/useNamingConvention: OAuth wire format
expires_in: 60,
interval: 1,
});
return;
}
if (url === "/token") {
const grantType = params.get("grant_type") ?? "";
if (grantType === "refresh_token") {
this.state.refreshGrants += 1;
if (params.get("refresh_token") !== this.state.currentRefresh) {
respond(400, { error: "invalid_grant" });
return;
}
} else if (!grantType.includes("device_code")) {
respond(400, { error: "unsupported_grant_type" });
return;
}
this.state.accessTokensIssued += 1;
const number = this.state.accessTokensIssued;
const refresh = `refresh-${number}`;
this.state.currentRefresh = refresh;
respond(200, {
// biome-ignore lint/style/useNamingConvention: OAuth wire format
access_token: `access-${number}`,
// biome-ignore lint/style/useNamingConvention: OAuth wire format
refresh_token: refresh,
// biome-ignore lint/style/useNamingConvention: OAuth wire format
expires_in: 3600,
});
return;
}
respond(404, {});
}
close(): void {
this.server?.close();
}
}
nodejs/__test__/package.test.ts
+2 -2
View File
@@ -5,8 +5,8 @@ import packageJson = require("../package.json");
describe("package metadata", () => {
it("requires Node.js type declarations compatible with the runtime", () => {
expect(packageJson.engines.node).toBe(">= 18");
expect(packageJson.peerDependencies["@types/node"]).toBe(">=18");
expect(packageJson.engines.node).toBe(">= 22");
expect(packageJson.peerDependencies["@types/node"]).toBe(">=22");
expect(packageJson.peerDependenciesMeta["@types/node"]).toEqual({
optional: true,
});
nodejs/__test__/remote.test.ts
+119 -24
View File
@@ -3,10 +3,14 @@
import * as http from "http";
import { RequestListener } from "http";
import packageJson = require("../package.json");
import {
ClientAuthMethod,
ClientConfig,
Connection,
ConnectionOptions,
OAuthConfig,
OAuthFlowType,
TlsConfig,
connect,
} from "../lancedb";
@@ -70,26 +74,28 @@ async function withMockDatabase(
try {
await callback(db);
} finally {
server.close();
// `close()` alone leaves the port bound until keep-alive sockets drain, so
// a single failing test would cascade into EADDRINUSE for every test after
// it. Destroy the connections and wait for the port to actually be free.
await new Promise<void>((resolve) => {
server.closeAllConnections();
server.close(() => resolve());
});
}
}
describe("remote connection", () => {
it("refuses materialized views before issuing any request", async () => {
const paths: string[] = [];
it("lists materialized views through the namespace route", async () => {
await withMockDatabase(
(req, res) => {
paths.push(req.url ?? "");
res.writeHead(404).end();
expect(req.method).toBe("GET");
expect(req.url).toBe("/v1/namespace/$/materialized_view/list");
res
.writeHead(200, { "content-type": "application/json" })
.end(JSON.stringify({ views: ["daily_sales"] }));
},
async (db) => {
await expect(db.openMaterializedView("secret_table")).rejects.toThrow(
/only on local databases/,
);
await expect(db.listMaterializedViews()).rejects.toThrow(
/only on local databases/,
);
expect(paths).toEqual([]);
expect(await db.listMaterializedViews()).toEqual(["daily_sales"]);
},
);
});
@@ -131,7 +137,7 @@ describe("remote connection", () => {
(req, res) => {
expect(req.headers["x-api-key"]).toEqual("fake");
expect(req.headers["user-agent"]).toEqual(
`LanceDB-Node-Client/${process.env.npm_package_version}`,
`LanceDB-Node-Client/${packageJson.version}`,
);
const body = JSON.stringify({ tables: [] });
@@ -435,6 +441,40 @@ describe("remote connection", () => {
]);
});
describe("OAuthConfig", () => {
it("should expose client auth method values", () => {
expect(ClientAuthMethod.None).toBe("none");
expect(ClientAuthMethod.ClientSecretBasic).toBe("client_secret_basic");
expect(ClientAuthMethod.ClientSecretPost).toBe("client_secret_post");
});
it("should accept a confidential client with basic auth", () => {
const config: OAuthConfig = {
issuerUrl: "https://issuer.example.com",
clientId: "client-id",
clientSecret: "secret",
scopes: ["openid"],
flow: OAuthFlowType.AuthorizationCode,
clientAuthMethod: ClientAuthMethod.ClientSecretBasic,
};
expect(config.clientAuthMethod).toBe(ClientAuthMethod.ClientSecretBasic);
});
it("should accept a public PKCE client without auth method or secret", () => {
const config: OAuthConfig = {
issuerUrl: "https://issuer.example.com",
clientId: "client-id",
scopes: ["openid"],
flow: OAuthFlowType.AuthorizationCode,
usePkce: true,
};
expect(config.clientSecret).toBeUndefined();
expect(config.clientAuthMethod).toBeUndefined();
});
});
describe("TlsConfig", () => {
it("should create TlsConfig with all fields", () => {
const tlsConfig: TlsConfig = {
@@ -932,6 +972,7 @@ describe("remote connection jobs surface", () => {
const { tableFromArrays, tableToIPC } = await import("apache-arrow");
const eventsTable = tableFromArrays({ state: ["created", "succeeded"] });
const eventsBody = Buffer.from(tableToIPC(eventsTable, "stream"));
const queryEventsPayloads: Record<string, unknown>[] = [];
await withMockDatabase(
(req, res) => {
@@ -960,6 +1001,16 @@ describe("remote connection jobs surface", () => {
);
}
} else if (req.url === "/v1/jobs/describe") {
if (payload["job_id"] === "job-2") {
res
.writeHead(200, { "Content-Type": "application/json" })
.end(
'{"job_id": "job-2", "job_type": "refresh_column", ' +
'"job_state": "DONE", "creation_ms": 2000, ' +
'"result": {"rows_assigned": 1000000}}',
);
return;
}
if (payload["job_id"] !== "job-1") {
res.writeHead(404).end("no such job");
return;
@@ -981,6 +1032,7 @@ describe("remote connection jobs surface", () => {
.writeHead(200, { "Content-Type": "application/json" })
.end('{"job_id": "job-1"}');
} else if (req.url === "/v1/jobs/query_events") {
queryEventsPayloads.push(payload);
res
.writeHead(200, {
"Content-Type": "application/vnd.apache.arrow.stream",
@@ -997,22 +1049,65 @@ describe("remote connection jobs surface", () => {
expect(jobs[0].state).toEqual("running");
expect(jobs[1].state).toEqual("finished");
const description = await db.getJob("job-1");
expect(description?.state).toEqual("failed");
expect(JSON.parse(description?.specJson ?? "")).toEqual({
column: "vec",
});
expect(description?.failure?.message).toEqual("worker died");
expect(await db.getJob("missing")).toBeNull();
expect(await db.cancelJob("job-1")).toBe(true);
expect(await db.cancelJob("missing")).toBe(false);
const history = await db.jobHistory("job-1");
expect(history.numRows).toEqual(2);
// Opening a job hands back a populated handle; a missing one rejects.
await expect(db.openJob("missing")).rejects.toThrow("not found");
const finished = await db.openJob("job-2");
expect(finished.state).toEqual("finished");
expect(finished.result).toEqual({
// biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
rows_assigned: 1000000,
});
const job = db.job("job-1");
const job = await db.openJob("job-1");
expect(job.id).toEqual("job-1");
// openJob already populated the handle; refresh() re-reads it.
expect(job.state).toEqual("failed");
await job.refresh();
expect(job.state).toEqual("failed");
expect(job.jobType).toEqual("create_index");
expect(job.creationMs).toEqual(1000);
expect(job.spec).toEqual({ column: "vec" });
expect(job.result).toBeNull();
expect(job.failure?.message).toEqual("worker died");
// The handle reaches its own events, supplying its job id.
const jobEvents = await job.events({
limit: 500,
filter: "state = 'claim_complete'",
});
expect(jobEvents.numRows).toEqual(2);
expect(queryEventsPayloads.pop()).toEqual({
// biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
job_id: "job-1",
limit: 500,
filter: "state = 'claim_complete'",
});
// Printing lays every known field out on its own line, with the JSON
// payloads indented rather than crammed onto one line.
expect(`${job}`).toEqual(
[
"Job(",
' id="job-1",',
' state="failed",',
' jobType="create_index",',
" creationMs=1000,",
" spec={",
' "column": "vec"',
" },",
" failure={",
' "phase": "execute",',
' "message": "worker died",',
' "retryable": true',
" },",
")",
].join("\n"),
);
expect(await job.status()).toEqual("failed");
await expect(job.wait()).rejects.toThrow("worker died");
},
nodejs/__test__/table.test.ts
+344 -7
View File
@@ -18,6 +18,7 @@ import {
Query,
Table,
VectorQuery,
blob,
connect,
tokenize,
} from "../lancedb";
@@ -52,6 +53,7 @@ import {
Operator,
instanceOfFullTextQuery,
} from "../lancedb/query";
import { LocalTable } from "../lancedb/table";
describe.each([arrow15, arrow16, arrow17, arrow18])(
"Given a table",
@@ -281,7 +283,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
numIndices: 0,
numRows: 3,
// Full on-disk size of the two data files, footers and metadata included.
totalBytes: 684,
totalBytes: 550,
});
// Index files count toward totalBytes too (only deletion files and
@@ -289,7 +291,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
await table.createIndex("id", { config: Index.btree() });
const statsWithIndex = await table.stats();
expect(statsWithIndex.numIndices).toBe(1);
expect(statsWithIndex.totalBytes).toBeGreaterThan(684);
expect(statsWithIndex.totalBytes).toBeGreaterThan(550);
});
it("should overwrite data if asked", async () => {
@@ -737,11 +739,12 @@ it("should query documents with LangChain PDF metadata", async () => {
describe("merge insert", () => {
let tmpDir: tmp.DirResult;
let conn: Connection;
let table: Table;
beforeEach(async () => {
tmpDir = tmp.dirSync({ unsafeCleanup: true });
const conn = await connect(tmpDir.name);
conn = await connect(tmpDir.name);
table = await conn.createTable("some_table", [
{ a: 1, b: "a" },
@@ -779,6 +782,38 @@ describe("merge insert", () => {
expect(result.map((row) => ({ ...row }))).toEqual(expected);
});
test("upsert on a composite key", async () => {
const composite = await conn.createTable("composite", [
{ shard: "a", id: 1, val: "x" },
{ shard: "a", id: 2, val: "y" },
{ shard: "b", id: 1, val: "z" },
]);
// ("a", 1) matches an existing row and updates it. ("b", 2) agrees with an
// existing row on each key column separately but on neither pair, so it is
// an insert.
const mergeInsertRes = await composite
.mergeInsert(["shard", "id"])
.whenMatchedUpdateAll()
.whenNotMatchedInsertAll()
.execute([
{ shard: "a", id: 1, val: "X" },
{ shard: "b", id: 2, val: "W" },
]);
expect(mergeInsertRes.numUpdatedRows).toBe(1);
expect(mergeInsertRes.numInsertedRows).toBe(1);
const result = (await composite.toArrow())
.toArray()
.sort((a, b) => a.shard.localeCompare(b.shard) || a.id - b.id);
expect(result.map((row) => ({ ...row }))).toEqual([
{ shard: "a", id: 1, val: "X" },
{ shard: "a", id: 2, val: "y" },
{ shard: "b", id: 1, val: "z" },
{ shard: "b", id: 2, val: "W" },
]);
});
test("conditional update", async () => {
const newData = [
{ a: 2, b: "x" },
@@ -2368,6 +2403,276 @@ describe("when dealing with versioning", () => {
});
});
describe("when dealing with blob columns", () => {
let tmpDir: tmp.DirResult;
beforeEach(() => {
tmpDir = tmp.dirSync({ unsafeCleanup: true });
});
afterEach(() => {
tmpDir.removeCallback();
});
it("discovers blob columns", async () => {
const { table } = await openBlobTable();
expect(await table.blobColumns()).toEqual(["image"]);
});
it("preserves order, duplicates, and nulls", async () => {
const { table, rowIds } = await openBlobTable();
const [alphaId, betaId, nullId] = rowIds;
const bytes = await table.fetchBlobs("image", [
betaId,
alphaId,
betaId,
nullId,
]);
expect(bytes.map((b) => (b == null ? null : b.toString()))).toEqual([
"beta",
"alpha",
"beta",
null,
]);
const files = await table.fetchBlobFiles("image", [
betaId,
nullId,
alphaId,
]);
expect(files.map((f) => f == null)).toEqual([false, true, false]);
});
it("reads full blob contents", async () => {
const { table, rowIds, alpha, beta } = await openBlobTable();
const bytes = await table.fetchBlobs("image", rowIds);
expect(bytes[0]!.equals(alpha)).toBe(true);
expect(bytes[1]!.equals(beta)).toBe(true);
const files = await table.fetchBlobFiles("image", rowIds);
expect(files[0]!.size()).toBe(BigInt(alpha.length));
expect(Buffer.from(await files[0]!.read()).toString()).toBe("alpha");
expect(Buffer.from(await files[1]!.read()).toString()).toBe("beta");
});
it("reads a half-open range", async () => {
const { table, rowIds } = await openBlobTable();
const files = await table.fetchBlobFiles("image", rowIds);
expect(Buffer.from(await files[0]!.readRange(0n, 2n)).toString()).toBe(
"al",
);
});
it("readRange does not move the cursor", async () => {
const { table, rowIds, alpha } = await openBlobTable();
const [handle] = await table.fetchBlobFiles("image", rowIds);
expect((await handle!.readRange(1n, 3n)).toString()).toBe("lp");
expect(await handle!.read()).toEqual(alpha);
expect(await handle!.read()).toEqual(Buffer.alloc(0));
});
it("fails when readRange end is past the blob size", async () => {
const { table, rowIds, alpha } = await openBlobTable();
const files = await table.fetchBlobFiles("image", rowIds);
await expect(
files[0]!.readRange(0n, BigInt(alpha.length + 1)),
).rejects.toThrow(/exceeds blob size/);
});
it("rejects fetchBlobs on a non-blob column", async () => {
const { table, rowIds } = await openBlobTable();
await expect(table.fetchBlobs("id", rowIds)).rejects.toThrow(/blob/i);
});
it("discovers and fetches nested blob columns", async () => {
const db = await connect(tmpDir.name);
const schema = new Schema([
new Field("id", new Int64(), true),
new Field("info", new Struct([blob("image")]), true),
]);
const payload = Buffer.from("nested");
const table = await db.createTable(
"nested_blobs",
[{ id: 1n, info: { image: payload } }],
{ schema },
);
expect(await table.blobColumns()).toEqual(["info.image"]);
const rows = await table.query().withRowId().toArray();
const bytes = await table.fetchBlobs("info.image", [
rows[0]._rowid as bigint,
]);
expect(bytes[0]!.equals(payload)).toBe(true);
});
it("creates and adds list blob columns", async () => {
const db = await connect(tmpDir.name);
const schema = new Schema([
new Field("id", new Int64(), true),
new Field("images", new List(blob("image")), true),
]);
const alpha = Buffer.from("alpha");
const beta = Buffer.from("beta");
const gamma = Buffer.from("gamma");
const table = await db.createTable(
"list_blobs",
[{ id: 1n, images: [alpha, beta] }],
{ schema },
);
await table.add([
{ id: 2n, images: null },
{ id: 3n, images: [gamma, null] },
{ id: 4n, images: [] },
]);
expect(await table.blobColumns()).toEqual(["images.image"]);
const rows = await table.query().toArray();
const byId = new Map(rows.map((row) => [Number(row.id), row]));
expect(descriptorSizes(byId.get(1)!.images)).toEqual([
alpha.length,
beta.length,
]);
expect(byId.get(2)!.images).toBeNull();
expect(descriptorSizes(byId.get(3)!.images)).toEqual([gamma.length, null]);
expect(Array.from(byId.get(4)!.images as Iterable<unknown>)).toHaveLength(
0,
);
});
it("creates and adds list struct blob columns", async () => {
const db = await connect(tmpDir.name);
const schema = new Schema([
new Field("id", new Int64(), true),
new Field(
"items",
new List(
new Field(
"item",
new Struct([new Field("name", new Utf8(), true), blob("image")]),
true,
),
),
true,
),
]);
const alpha = Buffer.from("nested-alpha");
const beta = Buffer.from("nested-beta");
const table = await db.createTable(
"list_struct_blobs",
[{ id: 1n, items: [{ name: "one", image: alpha }] }],
{ schema },
);
await table.add([
{
id: 2n,
items: [
{ name: "two", image: beta },
{ name: "three", image: null },
],
},
]);
const rows = await table.query().toArray();
const byId = new Map(rows.map((row) => [Number(row.id), row]));
expect(
descriptorSizes(
Array.from(byId.get(1)!.items as Iterable<{ image: unknown }>).map(
(item) => item.image,
),
),
).toEqual([alpha.length]);
expect(
descriptorSizes(
Array.from(byId.get(2)!.items as Iterable<{ image: unknown }>).map(
(item) => item.image,
),
),
).toEqual([beta.length, null]);
});
it("rejects blob fields inside a fixed-size list", async () => {
const db = await connect(tmpDir.name);
const schema = new Schema([
new Field("id", new Int64(), true),
new Field("frames", new FixedSizeList(2, blob("frame")), true),
]);
await expect(
db.createTable(
"fsl_blobs",
[{ id: 1n, frames: [Buffer.from("a"), Buffer.from("b")] }],
{ schema },
),
).rejects.toThrow(
"Blob fields inside FixedSizeList are not supported. Use List instead.",
);
});
it("rejects blob fields inside a nested fixed-size list", async () => {
const db = await connect(tmpDir.name);
const schema = new Schema([
new Field("id", new Int64(), true),
new Field(
"clip",
new Struct([
new Field("frames", new FixedSizeList(2, blob("frame")), true),
]),
true,
),
]);
await expect(
db.createTable(
"nested_fsl_blobs",
[
{
id: 1n,
clip: { frames: [Buffer.from("a"), Buffer.from("b")] },
},
],
{ schema },
),
).rejects.toThrow(
"Blob fields inside FixedSizeList are not supported. Use List instead.",
);
});
it("rejects an Arrow table with blob fields inside a fixed-size list", async () => {
const db = await connect(tmpDir.name);
const schema = new Schema([
new Field("id", new Int64(), true),
new Field("frames", new FixedSizeList(2, blob("frame")), true),
]);
await expect(
db.createTable("fsl_blobs_ipc", new ArrowTable(schema)),
).rejects.toThrow(
"Blob fields inside FixedSizeList are not supported. Use List instead.",
);
});
function descriptorSizes(values: unknown): (number | null)[] {
return Array.from(
values as Iterable<{ size?: bigint | number } | null>,
).map((value) => (value == null ? null : Number(value.size)));
}
async function openBlobTable() {
const db = await connect(tmpDir.name);
const schema = new Schema([
new Field("id", new Int64(), true),
blob("image"),
]);
const alpha = Buffer.from("alpha");
const beta = Buffer.from("beta");
const table = await db.createTable(
"blobs",
[
{ id: 1n, image: alpha },
{ id: 2n, image: beta },
{ id: 3n, image: null },
],
{ schema },
);
const rows = await table.query().withRowId().toArray();
const rowIdById = new Map(
rows.map((r) => [Number(r.id), r._rowid as bigint]),
);
const rowIds = [1, 2, 3].map((id) => rowIdById.get(id)!);
return { table, rowIds, alpha, beta };
}
});
describe("when dealing with tags", () => {
let tmpDir: tmp.DirResult;
beforeEach(() => {
@@ -2461,6 +2766,15 @@ describe("when dealing with tags", () => {
});
});
/** Returns a Date strictly later than every instant observed before the call. */
async function nextMillisecond(): Promise<Date> {
const start = Date.now();
while (Date.now() <= start) {
await new Promise((resolve) => setTimeout(resolve, 1));
}
return new Date();
}
describe("when optimizing a dataset", () => {
let tmpDir: tmp.DirResult;
let table: Table;
@@ -2483,9 +2797,14 @@ describe("when optimizing a dataset", () => {
});
it("cleanups old versions", async () => {
const stats = await table.optimize({ cleanupOlderThan: new Date() });
// Lance stores version timestamps with nanosecond precision while a JS
// Date only has millisecond precision. A cutoff captured in the same
// millisecond as the last commit would truncate to *before* that commit
// and leave it in place, so wait for the clock to tick over first.
const cutoff = await nextMillisecond();
const stats = await table.optimize({ cleanupOlderThan: cutoff });
expect(stats.prune.bytesRemoved).toBeGreaterThan(0);
expect(stats.prune.oldVersionsRemoved).toBe(3);
expect(stats.prune.oldVersionsRemoved).toBe(2);
});
it("delete unverified", async () => {
@@ -2506,6 +2825,24 @@ describe("when optimizing a dataset", () => {
});
});
it("passes cleanupOlderThan to the native binding as an absolute timestamp", async () => {
const optimize = jest.fn().mockResolvedValue({
compaction: {
filesAdded: 0,
filesRemoved: 0,
fragmentsAdded: 0,
fragmentsRemoved: 0,
},
prune: { bytesRemoved: 0, oldVersionsRemoved: 0 },
});
const table = new LocalTable({ optimize } as never);
const cutoff = new Date("2020-01-02T03:04:05.678Z");
await table.optimize({ cleanupOlderThan: cutoff, deleteUnverified: true });
expect(optimize).toHaveBeenCalledWith(cutoff.getTime(), true);
});
describe.each([arrow15, arrow16, arrow17, arrow18])(
"when optimizing a dataset",
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
@@ -3219,7 +3556,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
const db = await connect(tmpDir.name);
const data = [
{ text: "fa", vector: [0.1, 0.2, 0.3] },
{ text: "fo", vector: [0.4, 0.5, 0.6] },
{ text: "fo", vector: [0.4, 0.5, 0.6] }, // spellchecker:disable-line
{ text: "fob", vector: [0.4, 0.5, 0.6] },
{ text: "focus", vector: [0.4, 0.5, 0.6] },
{ text: "foo", vector: [0.4, 0.5, 0.6] },
@@ -3244,7 +3581,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
const resultSet = new Set(fuzzyResults.map((r) => r.text));
expect(resultSet.has("foo")).toBe(true);
expect(resultSet.has("fob")).toBe(true);
expect(resultSet.has("fo")).toBe(true);
expect(resultSet.has("fo")).toBe(true); // spellchecker:disable-line
expect(resultSet.has("food")).toBe(true);
const prefixResults = await table
nodejs/examples/package.json
+2 -1
View File
@@ -8,7 +8,8 @@
"//1": "--experimental-vm-modules is needed to run jest with sentence-transformers",
"//2": "--testEnvironment is needed to run jest with sentence-transformers",
"//3": "See: https://github.com/huggingface/transformers.js/issues/57",
"test": "node --experimental-vm-modules node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose",
"//4": "jest is invoked by its JS entry, not node_modules/.bin/jest: under pnpm that path is a shell shim, which `node` cannot execute",
"test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --testEnvironment jest-environment-node-single-context --verbose",
"lint": "biome check *.ts && biome format *.ts",
"lint-ci": "biome ci .",
"lint-fix": "biome check --write *.ts && pnpm format",
nodejs/jest.config.js
+9
View File
@@ -1,3 +1,12 @@
const path = require("node:path");
// Set the real process environment before Jest creates its sandbox and workers.
// Assigning process.env inside a test does not reach Rust's std::env.
process.env.LANCEDB_OAUTH_BROWSER =
process.platform === "win32"
? path.join(__dirname, "__test__/fixtures/oauth_browser.cmd")
: "/usr/bin/true";
/** @type {import('ts-jest').JestConfigWithTsJest} */
module.exports = {
preset: "ts-jest",
nodejs/lancedb/arrow.ts
+128 -3
View File
@@ -40,6 +40,7 @@ import {
} from "apache-arrow";
import { Buffers } from "apache-arrow/data";
import { typedArrayToArrowType } from "./arrow_type";
import { coerceBlobValue, isBlobField } from "./blob";
import { type EmbeddingFunction } from "./embedding/embedding_function";
import {
EmbeddingFunctionConfig,
@@ -71,6 +72,30 @@ export type FieldLike =
metadata?: Map<string, string>;
};
/**
* Create an Arrow field backed by LanceDB's JSON extension type.
*
* @param name - The field name.
* @param nullable - Whether the field accepts null values.
* @example
* ```ts
* import { connect, makeJsonField } from "@lancedb/lancedb";
* import { Schema } from "apache-arrow";
*
* const schema = new Schema([makeJsonField("metadata")]);
* const db = await connect("/path/to/database");
* await db.createTable("items", [{ metadata: '{"source":"api"}' }], { schema });
* ```
*/
export function makeJsonField(name: string, nullable = true): Field {
return new Field(
name,
new Utf8(),
nullable,
new Map([["ARROW:extension:name", "arrow.json"]]),
);
}
export type DataLike =
| import("apache-arrow").Data
| {
@@ -430,12 +455,14 @@ export function makeArrowTable(
throw new Error("A schema must be provided if data is empty");
} else {
schema = new Schema(schema.fields, schemaMetadata);
validateBlobSchema(schema);
return new ArrowTable(schema);
}
}
let inferredSchema = inferSchema(data, schema, opt);
inferredSchema = new Schema(inferredSchema.fields, schemaMetadata);
validateBlobSchema(inferredSchema);
const finalColumns: Record<string, Vector> = {};
for (const field of inferredSchema.fields) {
@@ -445,6 +472,35 @@ export function makeArrowTable(
return new ArrowTable(inferredSchema, finalColumns);
}
function validateBlobSchema(schema: Schema): void {
for (const field of schema.fields) {
validateBlobField(field);
}
}
function validateBlobField(field: Field): void {
if (
isFixedSizeList(field.type) &&
containsBlobField(field.type.children[0])
) {
throw new Error(
"Blob fields inside FixedSizeList are not supported. Use List instead.",
);
}
for (const child of field.type.children ?? []) {
validateBlobField(child);
}
}
function containsBlobField(field: Field): boolean {
if (isBlobField(field)) {
return true;
}
return (field.type.children ?? []).some((child: Field) =>
containsBlobField(child),
);
}
function isObject(value: unknown): value is Record<string, unknown> {
return (
typeof value === "object" &&
@@ -480,6 +536,32 @@ function transposeData(
path: string[] = [],
): Vector {
const valuesPath = [...path, field.name];
if (isBlobField(field) && field.type instanceof Struct) {
const blobRows = data.map((datum) =>
coerceBlobValue(valueAtPath(datum, valuesPath)),
);
const childVectors = field.type.children.map((child) => {
const values = blobRows.map((row) =>
row == null ? null : (row[child.name as "data" | "uri"] ?? null),
);
return makeVector(values, child.type, undefined, child.nullable);
});
const nullCount = blobRows.filter((row) => row === null).length;
const structData = makeData({
type: field.type,
length: blobRows.length,
nullCount,
nullBitmap:
nullCount > 0
? arrowUtil.packBools(blobRows.map((row) => row !== null))
: undefined,
children: childVectors.map((v) => v.data[0]),
});
return arrowMakeVector(structData);
}
if (isList(field.type) && containsBlobField(field.type.children[0])) {
return transposeListData(data, field, valuesPath);
}
const values = data.map((datum) => valueAtPath(datum, valuesPath));
if (field.type instanceof Struct) {
const childFields = field.type.children;
@@ -495,7 +577,7 @@ function transposeData(
nullCount > 0
? arrowUtil.packBools(values.map((value) => value !== null))
: undefined,
children: childVectors as unknown as ArrowData<DataType>[],
children: childVectors.map((v) => v.data[0]),
});
return arrowMakeVector(structData);
} else {
@@ -503,6 +585,48 @@ function transposeData(
}
}
function transposeListData(
data: Record<string, unknown>[],
field: Field,
valuesPath: string[],
): Vector {
const listType = field.type as List;
const childField = listType.children[0];
const lists = data.map((datum) => valueAtPath(datum, valuesPath));
const flattened: Record<string, unknown>[] = [];
const validity: boolean[] = [];
const offsets: number[] = [0];
for (const list of lists) {
if (list == null) {
validity.push(false);
offsets.push(flattened.length);
continue;
}
if (!Array.isArray(list)) {
throw new Error(`expected an array for list field '${field.name}'`);
}
validity.push(true);
for (const element of list) {
flattened.push({ [childField.name]: element });
}
offsets.push(flattened.length);
}
const childVector = transposeData(flattened, childField, []);
const nullCount = validity.filter((valid) => !valid).length;
return arrowMakeVector(
makeData({
type: listType,
length: lists.length,
nullCount,
nullBitmap: nullCount > 0 ? arrowUtil.packBools(validity) : undefined,
valueOffsets: Int32Array.from(offsets),
child: childVector.data[0],
}),
);
}
/**
* Create an empty Arrow table with the provided schema
*/
@@ -600,7 +724,7 @@ function makeVector(
}
if (values.length === 0) {
throw Error(
"makeVector requires at least one value or the type must be specfied",
"makeVector requires at least one value or the type must be specified",
);
}
const sampleValue = values.find((val) => val !== null && val !== undefined);
@@ -858,7 +982,7 @@ async function applyEmbeddings<T>(
* customized by the `embeddingDataType` property of the embedding function.
*
* If a schema is provided in `makeTableOptions` then it should include the
* embedding columns. If no schema is provded then embedding columns will
* embedding columns. If no schema is provided then embedding columns will
* be placed at the end of the table, after all of the input columns.
*/
export async function convertToTable(
@@ -952,6 +1076,7 @@ export async function fromTableToBuffer(
schema = sanitizeSchema(schema);
}
const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema);
validateBlobSchema(tableWithEmbeddings.schema);
const writer = RecordBatchFileWriter.writeAll(tableWithEmbeddings);
return Buffer.from(await writer.toUint8Array());
}
nodejs/lancedb/blob.ts
+236
View File
@@ -0,0 +1,236 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { Field, LargeBinary, Struct, Utf8 } from "apache-arrow";
import { BlobFile as NativeBlobFile } from "./native";
const BLOB_V2_EXTENSION_NAME = "lance.blob.v2";
const INLINE_SIZE_THRESHOLD_KEY = "lance-encoding:blob-inline-size-threshold";
const DEDICATED_SIZE_THRESHOLD_KEY =
"lance-encoding:blob-dedicated-size-threshold";
const PACK_FILE_SIZE_THRESHOLD_KEY =
"lance-encoding:blob-pack-file-size-threshold";
export type BlobInput = {
data: Buffer | Uint8Array | null;
uri: string | null;
};
export type BlobOptions = {
/** Defaults to true. */
nullable?: boolean;
/**
* Max payload bytes kept inline in the data file. Zero is allowed. Must be a
* safe integer.
*/
inlineSizeThreshold?: number;
/**
* Max payload bytes stored in a packed sidecar before a dedicated file. Must
* be a positive safe integer.
*/
dedicatedSizeThreshold?: number;
/**
* Max bytes in one packed sidecar before starting another. Must be a positive
* safe integer.
*/
packFileSizeThreshold?: number;
};
/**
* Declares a `lance.blob.v2` column.
*
* Query results are descriptors, not payload bytes. Use {@link Table.fetchBlobs}
* or {@link Table.fetchBlobFiles} to read bytes.
*
* @example
* ```ts
* import { readFile } from "node:fs/promises";
* import { Field, Int64, Schema } from "apache-arrow";
* import { blob, connect } from "@lancedb/lancedb";
*
* const db = await connect("./data");
* const video = await readFile("clip.mp4");
* const table = await db.createTable(
* "videos",
* [{ id: 1n, video }],
* {
* schema: new Schema([
* new Field("id", new Int64()),
* blob("video"),
* ]),
* },
* );
*
* const rows = await table.query().select(["id"]).withRowId().toArray();
* const rowIds = rows.map((row) => row._rowid as bigint);
* const bytes = await table.fetchBlobs("video", rowIds);
*
* const [handle] = await table.fetchBlobFiles("video", rowIds);
* const size = handle!.size();
* const header = await handle!.readRange(0n, size < 65536n ? size : 65536n);
* ```
*/
export function blob(name: string, options: BlobOptions = {}): Field {
const metadata = new Map<string, string>([
["ARROW:extension:name", BLOB_V2_EXTENSION_NAME],
]);
setThreshold(
metadata,
INLINE_SIZE_THRESHOLD_KEY,
"inlineSizeThreshold",
options.inlineSizeThreshold,
0,
);
setThreshold(
metadata,
DEDICATED_SIZE_THRESHOLD_KEY,
"dedicatedSizeThreshold",
options.dedicatedSizeThreshold,
1,
);
setThreshold(
metadata,
PACK_FILE_SIZE_THRESHOLD_KEY,
"packFileSizeThreshold",
options.packFileSizeThreshold,
1,
);
return new Field(
name,
new Struct([
new Field("data", new LargeBinary(), true),
new Field("uri", new Utf8(), true),
]),
options.nullable ?? true,
metadata,
);
}
/**
* Checks for the `lance.blob.v2` extension marker. Does not validate the
* field's storage type.
*/
export function isBlobField(field: Field): boolean {
return field.metadata?.get("ARROW:extension:name") === BLOB_V2_EXTENSION_NAME;
}
/**
* A lazy handle to blob bytes. Create one with {@link Table.fetchBlobFiles}.
*
* @hideconstructor
*/
export class BlobFile {
private readonly inner: NativeBlobFile;
private constructor(inner: NativeBlobFile) {
if (!(inner instanceof NativeBlobFile)) {
throw new Error("BlobFile handles come from Table.fetchBlobFiles");
}
this.inner = inner;
}
/** @ignore */
static fromNative(inner: NativeBlobFile): BlobFile {
return new BlobFile(inner);
}
/** Returns the blob size in bytes. */
size(): bigint {
return this.inner.size();
}
/**
* Reads from the cursor to the end and advances the cursor.
*
* A second call returns an empty buffer. {@link BlobFile.readRange} does
* not move the cursor.
*/
read(): Promise<Buffer> {
return this.inner.read();
}
/**
* Reads the half-open byte range `[start, end)`.
*
* Fails when `end` is past the blob size. Does not move the cursor.
*/
readRange(start: bigint, end: bigint): Promise<Buffer> {
return this.inner.readRange(start, end);
}
}
export function coerceBlobValue(value: unknown): BlobInput | null {
if (value == null) {
return null;
}
if (isBlobBytes(value)) {
return { data: value, uri: null };
}
if (ArrayBuffer.isView(value)) {
throw new Error("Blob data must be Buffer or Uint8Array");
}
if (typeof value === "string") {
if (value === "") {
throw new Error("Blob uri cannot be empty");
}
return { data: null, uri: value };
}
if (typeof value === "object") {
const record = value as Record<string, unknown>;
if (!("data" in record) && !("uri" in record)) {
throw new Error(
"Blob struct values must include a 'data' or 'uri' field",
);
}
const uri = record.uri;
if (uri === "") {
throw new Error("Blob uri cannot be empty");
}
if (uri != null && typeof uri !== "string") {
throw new Error(`Blob uri must be a string or null, got ${typeof uri}`);
}
const data = record.data;
if (data != null && !isBlobBytes(data)) {
throw new Error("Blob data must be Buffer, Uint8Array, or null");
}
const bytes = (data as Buffer | Uint8Array | null | undefined) ?? null;
const uriValue = uri ?? null;
if ((bytes == null) === (uriValue == null)) {
throw new Error(
"Blob struct values must set exactly one of 'data' or 'uri'",
);
}
return { data: bytes, uri: uriValue };
}
throw new Error(
"Blob column values must be Buffer, Uint8Array, a URI string, null, or { data?, uri? }",
);
}
function isBlobBytes(value: unknown): value is Buffer | Uint8Array {
return Buffer.isBuffer(value) || value instanceof Uint8Array;
}
function setThreshold(
metadata: Map<string, string>,
key: string,
optionName: string,
value: number | undefined,
minimum: number,
): void {
if (value === undefined) {
return;
}
if (!Number.isSafeInteger(value)) {
throw new Error(`${optionName} must be a safe integer`);
}
if (value < minimum) {
throw new Error(
minimum <= 0
? `${optionName} must be non-negative`
: `${optionName} must be positive`,
);
}
metadata.set(key, String(value));
}
nodejs/lancedb/catalog.ts
+103
View File
@@ -0,0 +1,103 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { Connection, LocalConnection } from "./connection";
import { HeaderProvider } from "./header";
import {
JsHeaderProvider,
ListDatabasesResponse,
Catalog as NativeCatalog,
CatalogOptions as NativeCatalogOptions,
} from "./native.js";
import { OAuthConfig } from "./oauth";
/** Options shared by a catalog and the database connections it returns. */
export interface CatalogOptions
extends Omit<NativeCatalogOptions, "oauthConfig"> {
oauthConfig?: OAuthConfig;
/** Called for each request to supply authentication headers. */
headerProvider?:
| HeaderProvider
| (() => Record<string, string> | Promise<Record<string, string>>);
}
export type { ListDatabasesResponse } from "./native.js";
/** A remote catalog manages databases through the server's root namespace. */
export class Catalog {
/** @hidden */
constructor(private readonly inner: NativeCatalog) {}
/** The root namespace endpoint. */
get uri(): string {
return this.inner.uri;
}
/** Create a database, or open an existing database when existOk is true. */
async createDatabase(
name: string,
options: { existOk?: boolean } = {},
): Promise<Connection> {
return new LocalConnection(
await this.inner.createDatabase(name, options.existOk),
);
}
/** Connect to an existing database by its logical name. */
async connectDatabase(name: string): Promise<Connection> {
return new LocalConnection(await this.inner.connectDatabase(name));
}
/** Drop an empty database. The server rejects nonempty databases. */
async dropDatabase(
name: string,
options: { ignoreMissing?: boolean } = {},
): Promise<void> {
await this.inner.dropDatabase(name, options.ignoreMissing);
}
/** List one page of databases; pass pageToken from a response for the next page. */
async listDatabases(
options: { limit?: number; pageToken?: string } = {},
): Promise<ListDatabasesResponse> {
if (
options.limit !== undefined &&
(!Number.isInteger(options.limit) ||
options.limit <= 0 ||
options.limit > 2147483647)
) {
throw new Error(
"Database list limit must be an integer between 1 and 2147483647",
);
}
return this.inner.listDatabases(options.limit, options.pageToken);
}
}
/**
* Connect to an HTTP(S) catalog endpoint. Catalog requests omit database-selection
* headers; opened database connections inherit authentication and client options.
*
* @example
* ```ts
* const catalog = await connectCatalog("https://my-server.example", { apiKey: "secret" });
* const db = await catalog.createDatabase("analytics", { existOk: true });
* const page = await catalog.listDatabases({ limit: 20 });
* ```
*/
export async function connectCatalog(
endpoint: string,
options: CatalogOptions = {},
): Promise<Catalog> {
const { headerProvider, ...nativeOptions } = options;
const provider = headerProvider
? new JsHeaderProvider(async () =>
typeof headerProvider === "function"
? headerProvider()
: headerProvider.getHeaders(),
)
: undefined;
return new Catalog(
await NativeCatalog.new(endpoint, nativeOptions, provider),
);
}
nodejs/lancedb/connection.ts
+59 -42
View File
@@ -1,7 +1,6 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { tableFromIPC } from "apache-arrow";
import {
Data,
SchemaLike,
@@ -16,6 +15,7 @@ import {
makeEmptyTable,
} from "./arrow";
import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
import { Job } from "./job";
import {
MaterializedView,
MaterializedViewSelect,
@@ -27,8 +27,6 @@ import type {
CreateNamespaceResponse,
DescribeNamespaceResponse,
DropNamespaceResponse,
Job,
JobDescription,
JobInfo,
ListNamespacesResponse,
ListTablesResponse,
@@ -322,13 +320,13 @@ export abstract class Connection {
/**
* Define a materialized view named `name` over the table `source`.
*
* The view is created empty, with the query recorded in its schema
* metadata; `view.refresh()` computes the rows. The view is a normal
* table: it can be queried, indexed and searched, and it appears in
* `tableNames`. The source table must have stable row ids (create it with
* The view is populated before creation returns. Set `withNoData` to create
* only its definition and empty backing table. The view is a normal table:
* it can be queried, indexed and searched, and it appears in `tableNames`.
* The source table must have stable row ids (create it with
* the `newTableEnableStableRowIds` storage option); they keep the view's
* provenance valid across source compactions and cannot be enabled after
* a table exists. Local databases only.
* a table exists.
*/
abstract createMaterializedView(
name: string,
@@ -337,6 +335,7 @@ export abstract class Connection {
select?: MaterializedViewSelect;
where?: string;
limit?: number;
withNoData?: boolean;
},
): Promise<MaterializedView>;
@@ -354,6 +353,30 @@ export abstract class Connection {
*/
abstract listMaterializedViews(): Promise<string[]>;
/**
* Drop the materialized view named `name`.
*
* The view may become unavailable before physical cleanup finishes. Use
* {@link dropMaterializedViewAsync} to retain and wait for the cleanup job.
*
* Rejects a table that exists but is not a materialized view.
*/
abstract dropMaterializedView(
name: string,
namespacePath?: string[],
): Promise<void>;
/**
* Start dropping the materialized view named `name` and return its cleanup
* job without waiting for completion.
*
* Rejects a table that exists but is not a materialized view.
*/
abstract dropMaterializedViewAsync(
name: string,
namespacePath?: string[],
): Promise<Job>;
abstract openTable(
name: string,
namespacePath?: string[],
@@ -557,24 +580,19 @@ export abstract class Connection {
): Promise<void>;
/**
* A {@link Job} handle for a server-side job by id.
* Open a server-side job by id, returning a handle with its record already
* populated. Rejects when the server has no such job, the way
* {@link Connection.openTable} does for a missing table.
*
* The handle is constructed without a server round trip; an unknown id
* surfaces when the handle is used. Dropping the handle has no effect on
* the job itself.
* The returned {@link Job} answers for its own state, specification,
* result, failure and event history, so there is no separate
* connection-level call for any of them.
*/
abstract job(jobId: string): Job;
abstract openJob(jobId: string): Promise<Job>;
/** List server-side jobs across the database's tables. */
abstract listJobs(): Promise<JobInfo[]>;
/**
* Describe a single server-side job by id.
*
* Resolves to `null` when the server has no such job.
*/
abstract getJob(jobId: string): Promise<JobDescription | null>;
/**
* Request cancellation of a server-side job by id.
*
@@ -582,13 +600,6 @@ export abstract class Connection {
* such job exists. Cancelling an already-terminal job is a no-op success.
*/
abstract cancelJob(jobId: string): Promise<boolean>;
/**
* The lifecycle event history of a server-side job, as an Arrow table.
*
* Lists history across all jobs when `jobId` is omitted.
*/
abstract jobHistory(jobId?: string): Promise<ArrowTable>;
}
/** @hideconstructor */
@@ -645,6 +656,7 @@ export class LocalConnection extends Connection {
select?: MaterializedViewSelect;
where?: string;
limit?: number;
withNoData?: boolean;
},
): Promise<MaterializedView> {
validateNonNegativeInteger(options?.limit, "limit");
@@ -654,6 +666,7 @@ export class LocalConnection extends Connection {
normalizeSelect(options?.select),
options?.where,
options?.limit,
options?.withNoData ?? false,
);
return new MaterializedView(new LocalTable(innerTable));
}
@@ -667,6 +680,22 @@ export class LocalConnection extends Connection {
return await this.inner.listMaterializedViews();
}
async dropMaterializedView(
name: string,
namespacePath?: string[],
): Promise<void> {
return this.inner.dropMaterializedView(name, namespacePath ?? []);
}
async dropMaterializedViewAsync(
name: string,
namespacePath?: string[],
): Promise<Job> {
return new Job(
await this.inner.dropMaterializedViewAsync(name, namespacePath ?? []),
);
}
async listTables(
namespacePathOrOptions?: string[] | Partial<ListTablesOptions>,
options?: Partial<ListTablesOptions>,
@@ -869,7 +898,7 @@ export class LocalConnection extends Connection {
}
async dropTableAsync(name: string, namespacePath?: string[]): Promise<Job> {
return this.inner.dropTableAsync(name, namespacePath ?? []);
return new Job(await this.inner.dropTableAsync(name, namespacePath ?? []));
}
async dropAllTables(namespacePath?: string[]): Promise<void> {
@@ -928,29 +957,17 @@ export class LocalConnection extends Connection {
);
}
job(jobId: string): Job {
return this.inner.job(jobId);
async openJob(jobId: string): Promise<Job> {
return new Job(await this.inner.openJob(jobId));
}
async listJobs(): Promise<JobInfo[]> {
return this.inner.listJobs();
}
async getJob(jobId: string): Promise<JobDescription | null> {
return this.inner.getJob(jobId);
}
async cancelJob(jobId: string): Promise<boolean> {
return this.inner.cancelJob(jobId);
}
async jobHistory(jobId?: string): Promise<ArrowTable> {
const buf = await this.inner.jobHistory(jobId);
if (buf.length === 0) {
return new ArrowTable();
}
return tableFromIPC(buf);
}
}
/**
nodejs/lancedb/index.ts
+23 -8
View File
@@ -72,11 +72,15 @@ export {
export {
makeArrowTable,
makeJsonField,
MakeArrowTableOptions,
Data,
VectorColumnOptions,
} from "./arrow";
export { blob, isBlobField, BlobFile } from "./blob";
export type { BlobOptions } from "./blob";
export {
Connection,
CreateTableOptions,
@@ -94,13 +98,9 @@ export {
RenameTableOptions,
} from "./connection";
export {
Job,
JobDescription,
JobFailureInfo,
JobInfo,
Session,
} from "./native.js";
export { JobFailureInfo, JobInfo, Session } from "./native.js";
export { Job, JobEventsOptions } from "./job";
export {
AutoQuery,
@@ -171,7 +171,15 @@ export {
TokenResponse,
} from "./header";
export { OAuthConfig, OAuthFlowType } from "./oauth";
export {
ClientAuthMethod,
OAuthConfig,
OAuthFlowType,
OAuthSession,
SessionLogout,
SessionStatus,
TokenCacheOptions,
} from "./oauth";
export { MergeInsertBuilder, WriteExecutionOptions } from "./merge";
@@ -621,3 +629,10 @@ export async function connectNamespace(
);
return new LocalConnection(nativeConn);
}
export {
Catalog,
CatalogOptions,
ListDatabasesResponse,
connectCatalog,
} from "./catalog";
nodejs/lancedb/indices.ts
+3 -3
View File
@@ -26,7 +26,7 @@ export interface IvfPqOptions {
* This value controls how much the vector is compressed during the quantization step.
* The more sub vectors there are the less the vector is compressed. The default is
* the dimension of the vector divided by 16. If the dimension is not evenly divisible
* by 16 we use the dimension divded by 8.
* by 16 we use the dimension divided by 8.
*
* The above two cases are highly preferred. Having 8 or 16 values per subvector allows
* us to use efficient SIMD instructions.
@@ -228,7 +228,7 @@ export interface HnswPqOptions {
* This value controls how much the vector is compressed during the quantization step.
* The more sub vectors there are the less the vector is compressed. The default is
* the dimension of the vector divided by 16. If the dimension is not evenly divisible
* by 16 we use the dimension divded by 8.
* by 16 we use the dimension divided by 8.
*
* The above two cases are highly preferred. Having 8 or 16 values per subvector allows
* us to use efficient SIMD instructions.
@@ -825,7 +825,7 @@ export interface IndexOptions {
/**
* Advanced index configuration
*
* This option allows you to specify a specfic index to create and also
* This option allows you to specify a specific index to create and also
* allows you to pass in configuration for training the index.
*
* See the static methods on Index for details on the various index types.
nodejs/lancedb/job.ts
+188
View File
@@ -0,0 +1,188 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { Table as ArrowTable, tableFromIPC } from "apache-arrow";
import { JobFailureInfo, Job as NativeJob } from "./native";
/** Which of a job's events {@link Job.events} returns. */
export interface JobEventsOptions {
/** Maximum event rows to return, up to the server maximum of 10,000. */
limit?: number;
/** SQL-like filter over the event columns. */
filter?: string;
}
/**
* A handle to an operation that may still be running.
*
* The operation may already be complete when the handle is created.
*
* The detail getters read what the handle last observed. Submitting an
* operation returns only a job id, so populating them eagerly would cost an
* extra round trip on every call:
*
* - {@link Job.refresh} and {@link Job.status} fetch the whole record.
* - {@link Job.wait} records the terminal state it establishes, but not the
* rest of the record.
* - Everything is null until one of those runs.
*
* @hideconstructor
*/
export class Job {
private readonly inner: NativeJob;
constructor(inner: NativeJob) {
this.inner = inner;
}
/**
* Identifies the operation on the server that is running it.
*
* Operations that run in this process have no server id. The value is
* opaque: parsing it or storing it to resume the job later is not supported.
*/
get id(): string | null {
return this.inner.id ?? null;
}
/** The last observed lifecycle state, without contacting the backend. */
get state(): string | null {
return this.inner.state ?? null;
}
/**
* The job's type, as the server names it. Null for an in-process job, which
* has no server-side record.
*/
get jobType(): string | null {
return this.inner.jobType ?? null;
}
/** When the job was created, in milliseconds since the epoch. */
get creationMs(): number | null {
return this.inner.creationMs ?? null;
}
/** The job-type-specific specification it was submitted with. */
// biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
get spec(): any | null {
return parseJson(this.inner.specJson);
}
/**
* The job-type-specific terminal result. Null until the job succeeds, so a
* job that never terminates reports its progress through {@link Job.events}
* instead.
*/
// biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
get result(): any | null {
return parseJson(this.inner.resultJson);
}
/** Why the job failed, when it failed and the server reports a reason. */
get failure(): JobFailureInfo | null {
return this.inner.failure ?? null;
}
/**
* The operation's current lifecycle state: "running", "finished", "failed",
* or "cancelled".
*
* A point snapshot; unlike {@link Job.wait} it does not block or reject on a
* terminal failure state. Also refreshes the getters above.
*/
async status(): Promise<string> {
return this.inner.status();
}
/** Wait until the operation reaches a terminal state. */
async wait(): Promise<void> {
return this.inner.wait();
}
/** Request cancellation. Cancelling a finished operation is a no-op. */
async cancel(): Promise<void> {
return this.inner.cancel();
}
/**
* Ask the backend for this job's current state, and for a server-side job
* its full record, then cache it for the getters above.
*/
async refresh(): Promise<void> {
return this.inner.refresh();
}
/**
* This job's recorded lifecycle events.
*
* Where the getters above report a terminal result only once the job reaches
* one, events are written as the job runs and outlive the workers that
* produced them. A distributed job records a `claim`/`claim_complete` pair
* per unit of work, each carrying `rows_processed`, so a job that never
* finishes still accounts for what it did.
*
* The server caps results at 1000 rows by default and 10,000 at most, and
* truncates without saying so, so pass `limit` for a job that emits an event
* per fragment. `filter` is a SQL-like expression over the `state`,
* `updated_by`, `emitted_from`, `emitted_by`, and `claim_entity` columns.
*/
async events(options?: JobEventsOptions): Promise<ArrowTable> {
const buf = await this.inner.events(options?.limit, options?.filter);
if (buf.length === 0) {
return new ArrowTable();
}
return tableFromIPC(buf);
}
/**
* Every field the handle currently knows, one per line, with the JSON
* payloads indented -- a refresh job's spec and result are the point of
* printing it.
*/
toString(): string {
if (this.state === null) {
const known = this.id === null ? "" : `id=${JSON.stringify(this.id)}, `;
return `Job(${known}not refreshed)`;
}
const fields: string[] = [];
if (this.id !== null) {
fields.push(`id=${JSON.stringify(this.id)}`);
}
fields.push(`state=${JSON.stringify(this.state)}`);
if (this.jobType !== null) {
fields.push(`jobType=${JSON.stringify(this.jobType)}`);
}
if (this.creationMs !== null) {
fields.push(`creationMs=${this.creationMs}`);
}
for (const [name, value] of [
["spec", this.spec],
["result", this.result],
] as const) {
if (value !== null) {
fields.push(`${name}=${indentJson(value)}`);
}
}
if (this.failure !== null) {
fields.push(`failure=${indentJson(this.failure)}`);
}
return `Job(${fields.map((field) => `\n${REPR_INDENT}${field},`).join("")}\n)`;
}
[Symbol.for("nodejs.util.inspect.custom")](): string {
return this.toString();
}
}
const REPR_INDENT = " ";
// biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
function indentJson(value: any): string {
return JSON.stringify(value, null, 4).replace(/\n/g, `\n${REPR_INDENT}`);
}
// biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
function parseJson(raw: string | null | undefined): any | null {
return raw === null || raw === undefined ? null : JSON.parse(raw);
}
nodejs/lancedb/materialized_view.ts
+22 -4
View File
@@ -19,6 +19,8 @@ export interface MaterializedViewDefinition {
limit?: number;
/** Source columns the projections and filter read. */
inputs: string[];
/** Namespace holding the source table; empty is the root namespace. */
sourceNamespace: string[];
}
/**
@@ -76,9 +78,22 @@ export function definitionFromMetadata(
if (raw === undefined) {
throw new Error(`Table '${name}' is not a materialized view`);
}
return definitionFromJson(raw, name);
}
/** @internal Parse the backend-independent definition returned by native code. */
export function definitionFromJson(
raw: string,
name: string,
): MaterializedViewDefinition {
// biome-ignore lint/suspicious/noExplicitAny: raw JSON
const value: any = JSON.parse(raw);
if (value.kind !== "select") {
// "namespaced_select" keeps older readers from resolving the source at root.
if (
value.kind !== undefined &&
value.kind !== "select" &&
value.kind !== "namespaced_select"
) {
throw new Error(
`materialized view '${name}' is defined by '${value.kind}', which this ` +
"version of lancedb cannot refresh",
@@ -103,6 +118,7 @@ export function definitionFromMetadata(
filter: value.filter ?? undefined,
limit,
inputs: value.inputs ?? [],
sourceNamespace: value.source_namespace ?? [],
};
}
@@ -130,10 +146,12 @@ export class MaterializedView {
return this.inner;
}
/** The query that defines the view, read from its stored schema. */
/** The query that defines the view. */
async definition(): Promise<MaterializedViewDefinition> {
const schema = await this.inner.schema();
return definitionFromMetadata(schema.metadata, this.name);
return definitionFromJson(
await this.inner.materializedViewDefinition(),
this.name,
);
}
/**
nodejs/lancedb/merge.ts
+1 -1
View File
@@ -27,7 +27,7 @@ export class MergeInsertBuilder {
* but that behavior is subject to change.
*
* An optional condition may be specified. If it is, then only
* matched rows that satisfy the condtion will be updated. Any
* matched rows that satisfy the condition will be updated. Any
* rows that do not satisfy the condition will be left as they
* are. Failing to satisfy the condition does not cause a
* "matched row" to become a "not matched" row.
nodejs/lancedb/oauth.ts
+242
View File
@@ -1,16 +1,79 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import {
OAuthConfig as NativeOAuthConfig,
OAuthSession as NativeOAuthSession,
} from "./native";
/**
* OAuth authentication flow types.
*/
export enum OAuthFlowType {
/** Client Credentials grant (service-to-service / M2M). */
ClientCredentials = "client_credentials",
/** Interactive Authorization Code grant, using PKCE by default. */
AuthorizationCode = "authorization_code",
/** Device Authorization grant for CLI and headless environments. */
DeviceCode = "device_code",
/** Azure Managed Identity via IMDS. */
AzureManagedIdentity = "azure_managed_identity",
}
/**
* Options for the persistent OAuth token cache.
*
* The cache is opt-in: it is only used when set as `tokenCache` on
* {@link OAuthConfig}. Only refresh tokens are persisted, in a private
* directory with owner-only permissions, so short-lived processes can reuse
* an authenticated session instead of re-prompting on every start.
*
* Multiple identities (issuer, client, scopes, resource, audience, flow, client authentication)
* get separate cache entries. Within one identity the most recent login wins.
*/
export interface TokenCacheOptions {
/**
* Directory that holds cached credentials. Defaults to
* `$XDG_CACHE_HOME/lancedb/oauth`, `$HOME/.cache/lancedb/oauth` on Unix,
* or `%LOCALAPPDATA%\\lancedb\\oauth` on Windows. The directory is created
* with owner-only permissions (`0700`) when missing.
*/
cacheDir?: string;
/**
* How long to wait for the cross-process refresh lock before failing, in
* seconds (default: 30).
*/
lockTimeoutSecs?: number;
}
/**
* How the client authenticates to the OAuth token endpoint.
*
* The method applies to every OAuth request that carries client
* authentication: client-credentials, authorization-code exchange,
* refresh-token, and device-authorization requests. The Azure managed
* identity flow ignores this option.
*/
export enum ClientAuthMethod {
/**
* No client authentication, for public clients using PKCE or the device
* flow. Cannot be combined with `clientSecret`.
*/
None = "none",
/**
* HTTP Basic authentication. This is the RFC 6749 recommended method and
* the normal default for confidential clients, including default Okta
* applications. Requires `clientSecret`.
*/
ClientSecretBasic = "client_secret_basic",
/**
* Credentials in the request body, for providers configured to require it.
* Requires `clientSecret`.
*/
ClientSecretPost = "client_secret_post",
}
/**
* OAuth configuration for LanceDB authentication.
*
@@ -31,6 +94,18 @@ export enum OAuthFlowType {
* };
* ```
*
* Providers requiring an explicit target can set `resource` and/or `audience`:
* ```typescript
* const targeted: OAuthConfig = {
* issuerUrl: "https://issuer.example.com",
* clientId: "app-id",
* clientSecret: "secret",
* scopes: ["read"],
* resource: "https://api.example.com",
* audience: "lancedb-api",
* };
* ```
*
* @example Azure Managed Identity:
* ```typescript
* const config: OAuthConfig = {
@@ -40,6 +115,21 @@ export enum OAuthFlowType {
* flow: OAuthFlowType.AzureManagedIdentity,
* };
* ```
*
* @example Authorization Code with PKCE:
* The authorization URL is written to stderr before LanceDB tries to open a
* browser, so it can be copied in headless environments.
* ```typescript
* const config: OAuthConfig = {
* issuerUrl: "https://login.microsoftonline.com/{tenant}/v2.0",
* clientId: "app-id",
* scopes: ["openid", "api://lancedb-api/access"],
* flow: OAuthFlowType.AuthorizationCode,
* };
* ```
*
* Device Authorization writes the verification URL and user code to stderr
* before polling begins.
*/
export interface OAuthConfig {
/**
@@ -58,12 +148,43 @@ export interface OAuthConfig {
*/
scopes: string[];
/**
* Resource indicator (RFC 8707), forwarded verbatim to authorization and token
* endpoints, including refresh requests. Must be an absolute URI without a
* fragment. Not supported for Azure managed identity.
*/
resource?: string;
/**
* Provider-specific audience, forwarded to authorization and token endpoints,
* including refresh requests. Not supported for Azure managed identity.
*/
audience?: string;
/** Authentication flow (default: ClientCredentials). */
flow?: OAuthFlowType;
/** Client secret (required for ClientCredentials). */
clientSecret?: string;
/**
* How the client authenticates to the token endpoint (default: auto).
* With a `clientSecret` the default is `ClientAuthMethod.ClientSecretBasic`,
* which matches the RFC 6749 recommendation and the default configuration
* of Okta confidential applications; without a secret the client is public
* and no client authentication is sent.
*/
clientAuthMethod?: ClientAuthMethod;
/** Loopback redirect URI for AuthorizationCode. */
redirectUri?: string;
/** Port for the AuthorizationCode loopback callback server (default: 8400). */
callbackPort?: number;
/** Protect AuthorizationCode with S256 PKCE (default: true). */
usePkce?: boolean;
/** Client ID for user-assigned managed identity (AzureManagedIdentity). */
managedIdentityClientId?: string;
@@ -73,4 +194,125 @@ export interface OAuthConfig {
* the TTL, each request refreshes the token.
*/
refreshBufferSecs?: number;
/**
* Opt in to the persistent token cache so short-lived processes reuse one
* session. Only refresh tokens are persisted. Only supported by
* AuthorizationCode and DeviceCode; Azure managed identity is rejected.
* Default: unset (memory only).
*/
tokenCache?: TokenCacheOptions;
}
/**
* Safe, non-secret view of a cached OAuth session, returned by
* {@link OAuthSession.status} and {@link OAuthSession.login}.
*/
export interface SessionStatus {
/**
* Whether a cached session exists that can obtain tokens without
* interactive authentication. Because access tokens are not persisted,
* this is `true` exactly when a refresh token is cached; the next
* connection refreshes with it rather than opening a browser or device
* prompt.
*/
refreshable: boolean;
/** Canonical issuer URL of the cached session. */
issuerUrl: string;
/** Client ID of the cached session. */
clientId: string;
/** Canonical (sorted, de-duplicated) scope set of the cached session. */
scopes: string[];
/** Resource indicator used to obtain the cached session, if configured. */
resource?: string;
/** Provider-specific audience used to obtain the cached session, if configured. */
audience?: string;
/** Flow that produced the cached session. */
flow: string;
/** When the cached session was obtained, as Unix seconds. */
obtainedAt?: number;
}
/** Result of {@link OAuthSession.logout}. */
export interface SessionLogout {
/**
* Whether a cached credential was removed. `false` means no matching
* session was cached; logout is idempotent.
*/
removed: boolean;
}
/**
* Explicit OAuth session lifecycle for the persistent token cache: eager
* `login`, non-secret `status`, and local `logout`.
*
* A session is built from the same {@link OAuthConfig} used to connect
* (including its `tokenCache` options). A connection created with the same
* configuration shares the cache, so logging in here prepares tokens for
* later processes without any database request.
*
* `login` always runs the configured interactive flow and replaces the cached
* session (the most recent login wins). `logout` removes only the local
* credential; it does not revoke anything with the provider and does not sign
* out of a browser SSO session.
*
* @example
* ```typescript
* const config: OAuthConfig = {
* issuerUrl: "https://issuer.example.com",
* clientId: "my-app",
* scopes: ["openid", "offline_access"],
* flow: OAuthFlowType.DeviceCode,
* tokenCache: { cacheDir: "/tmp/my-app/oauth-cache" },
* };
* const session = new OAuthSession(config);
* const status = await session.login();
* ```
*/
export class OAuthSession {
private readonly inner: NativeOAuthSession;
/** Create a session manager for the given OAuth configuration. */
constructor(config: OAuthConfig) {
this.inner = new NativeOAuthSession(config as unknown as NativeOAuthConfig);
}
/**
* Eagerly run the configured authentication flow and store the session.
*
* A successful login always replaces any prior cached session for this
* identity; if the provider does not issue a refresh token (for example
* without `offline_access`), the previous record is removed and the status
* reports `refreshable == false`.
*/
async login(): Promise<SessionStatus> {
return this.inner.login();
}
/**
* Report whether a matching cached session exists, with safe metadata.
*
* This never contacts the identity provider and never exposes token values.
*/
async status(): Promise<SessionStatus> {
return this.inner.status();
}
/**
* Remove the matching local cached credential.
*
* This only deletes the local cache entry. It does not revoke the refresh
* token with the provider and does not sign out of a browser SSO session.
* Repeated calls succeed; `removed` reports whether a credential existed.
*/
async logout(): Promise<SessionLogout> {
return this.inner.logout();
}
}
nodejs/lancedb/sanitize.ts
+1 -1
View File
@@ -3,7 +3,7 @@
// The utilities in this file help sanitize data from the user's arrow
// library into the types expected by vectordb's arrow library. Node
// generally allows for mulitple versions of the same library (and sometimes
// generally allows for multiple versions of the same library (and sometimes
// even multiple copies of the same version) to be installed at the same
// time. However, arrow-js uses instanceof which expected that the input
// comes from the exact same library instance. This is not always the case
nodejs/lancedb/table.ts
+116 -45
View File
@@ -17,8 +17,10 @@ import {
tableFromIPC,
} from "./arrow";
import { BlobFile } from "./blob";
import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
import { IndexOptions } from "./indices";
import { Job } from "./job";
import { MergeInsertBuilder } from "./merge";
import {
AddColumnsResult,
@@ -30,7 +32,6 @@ import {
DropColumnsResult,
IndexConfig,
IndexStatistics,
Job,
LsmStats,
Branches as NativeBranches,
OptimizeStats,
@@ -147,7 +148,8 @@ export interface OptimizeOptions {
* olderThan.setDate(olderThan.getDate() - 1));
* tbl.optimize({cleanupOlderThan: olderThan});
*
* // Delete all versions except the current version
* // Delete versions committed before this point. Versions created by the
* // optimize call itself are newer than the cutoff and will be retained.
* tbl.optimize({cleanupOlderThan: new Date()});
*/
cleanupOlderThan: Date;
@@ -313,7 +315,7 @@ export abstract class Table {
* Note: if your condition is something like "some_id_column == 7" and
* you are updating many rows (with different ids) then you will get
* better performance with a single [`merge_insert`] call instead of
* repeatedly calilng this method.
* repeatedly calling this method.
* @param {Map<string, string> | Record<string, string>} updates - the
* columns to update
* @returns {Promise<UpdateResult>} A promise that resolves to an object
@@ -510,6 +512,35 @@ export abstract class Table {
*/
abstract takeRowIds(rowIds: readonly (bigint | number)[]): TakeQuery;
/**
* Blob v2 columns, including nested dotted paths.
*/
abstract blobColumns(): Promise<string[]>;
/**
* Bytes for `column` at row IDs from {@link Query.withRowId}.
*
* Reads the table's current checkout. IDs from another version can fail after
* compaction unless stable row ids are enabled. Results keep input order and
* duplicates. Null blobs are `null`. Empty blobs are empty buffers.
*/
abstract fetchBlobs(
column: string,
rowIds: readonly (bigint | number)[],
): Promise<(Buffer | null)[]>;
/**
* Opens lazy blob handles for `column` at the given row IDs using the
* table's current checkout.
*
* Preserves input order, duplicates, and nulls. Use this for large payloads.
* See {@link Table.fetchBlobs} for row-ID validity across versions.
*/
abstract fetchBlobFiles(
column: string,
rowIds: readonly (bigint | number)[],
): Promise<(BlobFile | null)[]>;
/**
* Create a search query to find the nearest neighbors
* of the given query
@@ -542,10 +573,10 @@ export abstract class Table {
* {@link Table#refreshColumn}. Declaring one therefore costs the same on a
* large table as on an empty one.
*
* A refresh does not revisit rows it has already filled, so mutating an
* input leaves the value computed at fill time; recomputing means dropping
* the column and declaring it again. While a declaration reads a column,
* that column cannot be renamed, retyped or dropped.
* A refresh also recomputes the rows whose inputs changed since they were
* computed, so a mutated input is reflected by the next refresh. While a
* declaration reads a column, that column cannot be renamed, retyped or
* dropped.
*
* On LanceDB Cloud and Enterprise the expression is planned by the
* server, and the refresh runs as a server job -- see
@@ -576,10 +607,10 @@ export abstract class Table {
/**
* Fill the rows of a computed column that hold no value yet.
*
* Rows appended since the last refresh are filled by the next one; rows
* already filled are left as they are, so the call is idempotent and does
* not observe a mutated input. Local tables only: a remote refresh runs
* as a server job, through {@link Table#refreshColumnAsync}.
* Rows appended since the last refresh are filled by the next one, and
* rows whose inputs changed since they were computed are recomputed;
* everything else is left as it is. Local tables only: a remote refresh
* runs as a server job, through {@link Table#refreshColumnAsync}.
* @param {string} column The name of the computed column to fill.
* @returns {Promise<RefreshColumnResult>} A promise that resolves to the
* number of rows filled and the new version number of the table.
@@ -609,7 +640,7 @@ export abstract class Table {
* Recompute this table's contents from its materialized-view definition.
*
* Plumbing for {@link MaterializedView.refresh}, which is the way to call
* it: rejects tables that carry no view definition. Local tables only.
* it: rejects tables that carry no view definition.
* @ignore
*/
abstract refreshMaterializedView(
@@ -617,6 +648,9 @@ export abstract class Table {
sourceVersion?: number,
): Promise<RefreshMaterializedViewResult>;
/** @ignore */
abstract materializedViewDefinition(): Promise<string>;
/**
* Alter the name or nullability of columns.
* @param {ColumnAlteration[]} columnAlterations One or more alterations to
@@ -919,6 +953,16 @@ export abstract class Table {
/** Return the table as an arrow table */
abstract toArrow(): Promise<ArrowTable>;
/**
* Create a {@link MergeInsertBuilder}, which combines new data with the
* existing table in a single transaction — inserting, updating and deleting
* rows depending on how they match.
*
* @param on - The column, or columns, to match source rows against target
* rows on. Typically a key or id column. Several columns match on the
* composite key: a source row updates a target row only when it agrees on
* every one of them.
*/
abstract mergeInsert(on: string | string[]): MergeInsertBuilder;
/** List all the stats of a specified index
@@ -1114,13 +1158,15 @@ export class LocalTable extends Table {
): Promise<Job> {
// biome-ignore lint/suspicious/noExplicitAny: skip
const nativeIndex = (options?.config as any)?.inner;
return await this.inner.createIndexAsync(
nativeIndex,
column,
options?.replace,
options?.waitTimeoutSeconds,
options?.name,
options?.train,
return new Job(
await this.inner.createIndexAsync(
nativeIndex,
column,
options?.replace,
options?.waitTimeoutSeconds,
options?.name,
options?.train,
),
);
}
@@ -1148,23 +1194,34 @@ export class LocalTable extends Table {
}
takeRowIds(rowIds: readonly (bigint | number)[]): TakeQuery {
const ids = rowIds.map((id) => {
if (typeof id === "bigint") {
return id;
}
if (!Number.isInteger(id)) {
throw new Error("Row id must be an integer (or bigint)");
}
if (id < 0) {
throw new Error("Row id cannot be negative");
}
if (!Number.isSafeInteger(id)) {
throw new Error("Row id is too large for number; use bigint instead");
}
return BigInt(id);
});
return new TakeQuery(this.inner.takeRowIds(rowIdsToBigInts(rowIds)));
}
return new TakeQuery(this.inner.takeRowIds(ids));
blobColumns(): Promise<string[]> {
return this.inner.blobColumns();
}
async fetchBlobs(
column: string,
rowIds: readonly (bigint | number)[],
): Promise<(Buffer | null)[]> {
const values = await this.inner.fetchBlobs(column, rowIdsToBigInts(rowIds));
// N-API Option maps missing values to undefined. Collapse those to null.
return values.map((value) => value ?? null);
}
async fetchBlobFiles(
column: string,
rowIds: readonly (bigint | number)[],
): Promise<(BlobFile | null)[]> {
const files = await this.inner.fetchBlobFiles(
column,
rowIdsToBigInts(rowIds),
);
// N-API Option maps missing values to undefined. Collapse those to null.
return files.map((file) =>
file == null ? null : BlobFile.fromNative(file),
);
}
query(): Query {
@@ -1303,7 +1360,7 @@ export class LocalTable extends Table {
}
async refreshColumnAsync(column: string): Promise<Job> {
return await this.inner.refreshColumnAsync(column);
return new Job(await this.inner.refreshColumnAsync(column));
}
async refreshMaterializedView(
@@ -1313,6 +1370,10 @@ export class LocalTable extends Table {
return await this.inner.refreshMaterializedView(full, sourceVersion);
}
async materializedViewDefinition(): Promise<string> {
return await this.inner.materializedViewDefinition();
}
async alterColumns(
columnAlterations: ColumnAlteration[],
): Promise<AlterColumnsResult> {
@@ -1433,16 +1494,8 @@ export class LocalTable extends Table {
}
async optimize(options?: Partial<OptimizeOptions>): Promise<OptimizeStats> {
let cleanupOlderThanMs;
if (
options?.cleanupOlderThan !== undefined &&
options?.cleanupOlderThan !== null
) {
cleanupOlderThanMs =
new Date().getTime() - options.cleanupOlderThan.getTime();
}
return await this.inner.optimize(
cleanupOlderThanMs,
options?.cleanupOlderThan?.getTime(),
options?.deleteUnverified,
);
}
@@ -1721,3 +1774,21 @@ export class Branches {
)) as unknown as CherryPickResult;
}
}
function rowIdsToBigInts(rowIds: readonly (bigint | number)[]): bigint[] {
return rowIds.map((id) => {
if (typeof id === "bigint") {
return id;
}
if (!Number.isInteger(id)) {
throw new Error("Row id must be an integer (or bigint)");
}
if (id < 0) {
throw new Error("Row id cannot be negative");
}
if (!Number.isSafeInteger(id)) {
throw new Error("Row id is too large for number; use bigint instead");
}
return BigInt(id);
});
}
nodejs/npm/darwin-arm64/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-darwin-arm64",
"version": "0.38.0-beta.12",
"version": "0.40.0-beta.1",
"os": ["darwin"],
"cpu": ["arm64"],
"main": "lancedb.darwin-arm64.node",
nodejs/npm/linux-arm64-gnu/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-gnu",
"version": "0.38.0-beta.12",
"version": "0.40.0-beta.1",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-gnu.node",
nodejs/npm/linux-arm64-musl/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-musl",
"version": "0.38.0-beta.12",
"version": "0.40.0-beta.1",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-musl.node",
nodejs/npm/linux-x64-gnu/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-gnu",
"version": "0.38.0-beta.12",
"version": "0.40.0-beta.1",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-gnu.node",
nodejs/npm/linux-x64-musl/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-musl",
"version": "0.38.0-beta.12",
"version": "0.40.0-beta.1",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-musl.node",
nodejs/npm/win32-arm64-msvc/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-arm64-msvc",
"version": "0.38.0-beta.12",
"version": "0.40.0-beta.1",
"os": [
"win32"
],
nodejs/npm/win32-x64-msvc/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-x64-msvc",
"version": "0.38.0-beta.12",
"version": "0.40.0-beta.1",
"os": ["win32"],
"cpu": ["x64"],
"main": "lancedb.win32-x64-msvc.node",
nodejs/package-lock.json
Generated
-11106
View File
File diff suppressed because it is too large Load Diff
nodejs/package.json
+5 -5
View File
@@ -11,7 +11,7 @@
"ann"
],
"private": false,
"version": "0.38.0-beta.12",
"version": "0.40.0-beta.1",
"main": "dist/index.js",
"exports": {
".": "./dist/index.js",
@@ -44,7 +44,7 @@
"@biomejs/biome": "^1.7.3",
"@jest/globals": "^29.7.0",
"@napi-rs/cli": "3.7.0",
"@opentelemetry/sdk-metrics": "^1.30.0",
"@opentelemetry/sdk-metrics": "^2.10.0",
"@types/axios": "^0.14.0",
"@types/jest": "^29.1.2",
"@types/node": "22.7.4",
@@ -56,7 +56,7 @@
"eslint": "^8.57.0",
"jest": "^29.7.0",
"shx": "^0.3.4",
"tmp": "^0.2.3",
"tmp": "^0.2.7",
"ts-jest": "^29.1.2",
"typedoc": "0.26.4",
"typedoc-plugin-markdown": "4.2.1",
@@ -67,7 +67,7 @@
"timeout": "3m"
},
"engines": {
"node": ">= 18"
"node": ">= 22"
},
"packageManager": "pnpm@11.1.1",
"cpu": ["x64", "arm64"],
@@ -101,7 +101,7 @@
"openai": "4.29.2"
},
"peerDependencies": {
"@types/node": ">=18",
"@types/node": ">=22",
"apache-arrow": ">=15.0.0 <=18.1.0"
},
"peerDependenciesMeta": {
nodejs/pnpm-lock.yaml
Generated
+600 -444
View File
File diff suppressed because it is too large Load Diff
nodejs/pnpm-workspace.yaml
+38
View File
@@ -16,3 +16,41 @@ allowBuilds:
onnxruntime-node: true
protobufjs: true
sharp: true
minimumReleaseAgeExclude:
- protobufjs@7.5.8
- tmp@0.2.6
- form-data@4.0.6
- tar@7.5.16
- markdown-it@14.1.2
- linkify-it@5.0.1
- js-yaml@3.15.0
- js-yaml@4.1.2
- protobufjs@7.6.1
- protobufjs@7.6.3
- '@babel/core@7.29.1'
- axios@1.18.0
- brace-expansion@2.1.2
- brace-expansion@1.1.16
- js-yaml@4.3.0
- tar@7.5.18
- tar@7.5.19
- tar@7.5.17
- protobufjs@7.6.5
- linkify-it@5.0.2
- sharp@0.35.0
- brace-expansion@1.1.17
- brace-expansion@2.1.3
- brace-expansion@2.1.4
- brace-expansion@1.1.18
- js-yaml@3.15.1
- js-yaml@4.3.1
- tar@7.5.21
- '@opentelemetry/core@2.8.0'
# @huggingface/transformers pins sharp ^0.33.5 and no released version has moved
# past ^0.34.5, all of which inherit the libvips CVEs in GHSA-f88m-g3jw-g9cj.
# Force the patched line. sharp is only reached by transformers' image pipeline,
# which LanceDB's text embedding function never uses.
overrides:
sharp: ^0.35.4
nodejs/src/blob.rs
+95
View File
@@ -0,0 +1,95 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::ops::Range;
use std::sync::Arc;
use arrow_array::{Array, LargeBinaryArray};
use lancedb::blob::BlobFile as LanceBlobFile;
use napi::bindgen_prelude::*;
use napi_derive::napi;
use crate::error::convert_error;
#[napi]
pub struct BlobFile {
inner: Arc<LanceBlobFile>,
}
impl BlobFile {
pub(crate) fn new(inner: LanceBlobFile) -> Self {
Self {
inner: Arc::new(inner),
}
}
}
#[napi]
impl BlobFile {
#[napi]
pub fn size(&self) -> BigInt {
BigInt::from(self.inner.size())
}
#[napi]
pub async fn read(&self) -> napi::Result<Buffer> {
let bytes = self.inner.read().await.map_err(|err| convert_error(&err))?;
Ok(Buffer::from(bytes.as_ref()))
}
#[napi]
pub async fn read_range(&self, start: BigInt, end: BigInt) -> napi::Result<Buffer> {
let range = bigint_range(start, end)?;
let bytes = self
.inner
.read_range(range)
.await
.map_err(|err| convert_error(&err))?;
Ok(Buffer::from(bytes.as_ref()))
}
}
fn bigint_range(start: BigInt, end: BigInt) -> napi::Result<Range<u64>> {
let start = parse_u64(start, "start")?;
let end = parse_u64(end, "end")?;
if start > end {
return Err(napi::Error::from_reason(format!(
"invalid blob range: start ({start}) > end ({end})"
)));
}
Ok(start..end)
}
fn parse_u64(value: BigInt, name: &str) -> napi::Result<u64> {
let (negative, value, lossless) = value.get_u64();
if negative {
return Err(napi::Error::from_reason(format!(
"{name} cannot be negative"
)));
}
if !lossless {
return Err(napi::Error::from_reason(format!(
"{name} is too large to fit in u64"
)));
}
Ok(value)
}
pub fn parse_row_ids(row_ids: Vec<BigInt>) -> napi::Result<Vec<u64>> {
row_ids
.into_iter()
.map(|id| parse_u64(id, "row id"))
.collect()
}
pub fn copy_blob_buffers(array: LargeBinaryArray) -> Vec<Option<Buffer>> {
(0..array.len())
.map(|i| {
if array.is_null(i) {
None
} else {
Some(Buffer::from(array.value(i).to_vec()))
}
})
.collect()
}
nodejs/src/catalog.rs
+123
View File
@@ -0,0 +1,123 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::sync::Arc;
use std::time::Duration;
use lancedb::catalog::{
CatalogConnection, CreateDatabaseRequest, DropDatabaseRequest, ListDatabasesRequest,
};
use napi::bindgen_prelude::*;
use napi_derive::napi;
use crate::connection::Connection;
use crate::error::NapiErrorExt;
use crate::header::JsHeaderProvider;
use crate::remote::{ClientConfig, OAuthConfig};
#[napi(object)]
pub struct CatalogOptions {
pub api_key: Option<String>,
pub client_config: Option<ClientConfig>,
/// SQL service endpoint inherited by database connections.
pub sql_host_override: Option<String>,
pub read_consistency_interval: Option<f64>,
pub oauth_config: Option<OAuthConfig>,
}
#[napi(object)]
pub struct ListDatabasesResponse {
pub databases: Vec<String>,
pub page_token: Option<String>,
}
#[napi]
pub struct Catalog {
inner: CatalogConnection,
}
#[napi]
impl Catalog {
#[napi(factory)]
pub async fn new(
endpoint: String,
options: CatalogOptions,
header_provider: Option<&JsHeaderProvider>,
) -> Result<Self> {
let mut builder = lancedb::connect_catalog(endpoint);
if let Some(key) = options.api_key {
builder = builder.api_key(key);
}
let mut config: lancedb::remote::ClientConfig =
options.client_config.unwrap_or_default().into();
if let Some(provider) = header_provider {
config.header_provider = Some(Arc::new(provider.clone()));
}
builder = builder.client_config(config);
if let Some(endpoint) = options.sql_host_override {
builder = builder.sql_host_override(endpoint);
}
if let Some(interval) = options.read_consistency_interval {
let interval = Duration::try_from_secs_f64(interval).map_err(|err| {
Error::from_reason(format!("Invalid read consistency interval: {err}"))
})?;
builder = builder.read_consistency_interval(interval);
}
if let Some(oauth) = options.oauth_config {
builder = builder.oauth_config(oauth.try_into().default_error()?);
}
Ok(Self {
inner: builder.execute().await.default_error()?,
})
}
#[napi(getter)]
pub fn uri(&self) -> String {
self.inner.uri().to_string()
}
#[napi]
pub async fn create_database(
&self,
name: String,
exist_ok: Option<bool>,
) -> Result<Connection> {
self.inner
.create_database(CreateDatabaseRequest::new(name).exist_ok(exist_ok.unwrap_or(false)))
.await
.map(Connection::inner_new)
.default_error()
}
#[napi]
pub async fn connect_database(&self, name: String) -> Result<Connection> {
self.inner
.connect_database(name)
.await
.map(Connection::inner_new)
.default_error()
}
#[napi]
pub async fn drop_database(&self, name: String, ignore_missing: Option<bool>) -> Result<()> {
self.inner
.drop_database(
DropDatabaseRequest::new(name).ignore_missing(ignore_missing.unwrap_or(false)),
)
.await
.default_error()
}
#[napi]
pub async fn list_databases(
&self,
limit: Option<u32>,
page_token: Option<String>,
) -> Result<ListDatabasesResponse> {
let mut request = ListDatabasesRequest::default();
request.limit = limit;
request.page_token = page_token;
let response = self.inner.list_databases(request).await.default_error()?;
Ok(ListDatabasesResponse {
databases: response.databases,
page_token: response.page_token,
})
}
}
nodejs/src/connection.rs
+41 -46
View File
@@ -308,6 +308,7 @@ impl Connection {
projections: Option<Vec<Vec<String>>>,
filter: Option<String>,
limit: Option<i64>,
with_no_data: bool,
) -> napi::Result<Table> {
let mut builder = self.get_inner()?.create_materialized_view(name, source);
if let Some(projections) = projections {
@@ -328,6 +329,7 @@ impl Connection {
.map_err(|_| napi::Error::from_reason("limit must be a non-negative integer"))?;
builder = builder.limit(limit);
}
builder = builder.with_no_data(with_no_data);
let view = builder.execute().await.default_error()?;
Ok(Table::new(view.table().clone()))
}
@@ -349,7 +351,37 @@ impl Connection {
.list_materialized_views()
.await
.default_error()?;
Ok(views.into_iter().map(|v| v.name).collect())
Ok(views)
}
/// Drop a materialized view.
#[napi(catch_unwind)]
pub async fn drop_materialized_view(
&self,
name: String,
namespace_path: Option<Vec<String>>,
) -> napi::Result<()> {
let ns = namespace_path.unwrap_or_default();
self.get_inner()?
.drop_materialized_view(&name, &ns)
.await
.default_error()
}
/// Start dropping a materialized view and return its cleanup job.
#[napi(catch_unwind)]
pub async fn drop_materialized_view_async(
&self,
name: String,
namespace_path: Option<Vec<String>>,
) -> napi::Result<crate::job::Job> {
let ns = namespace_path.unwrap_or_default();
let job = self
.get_inner()?
.drop_materialized_view_async(&name, &ns)
.await
.default_error()?;
Ok(crate::job::Job::new(job))
}
#[napi(catch_unwind)]
@@ -442,13 +474,15 @@ impl Connection {
self.get_inner()?.drop_all_tables(&ns).await.default_error()
}
/// A `Job` handle for a server-side job by id.
/// Open a server-side job by id, returning a handle with its record
/// already populated. Rejects when the server has no such job.
///
/// The handle is constructed without a server round trip; an unknown id
/// surfaces when the handle is used.
#[napi]
pub fn job(&self, job_id: String) -> napi::Result<crate::job::Job> {
let job = self.get_inner()?.job(job_id).default_error()?;
/// The returned handle answers for its own state, specification, result,
/// failure and event history, so there is no separate connection-level
/// call for any of them.
#[napi(catch_unwind)]
pub async fn open_job(&self, job_id: String) -> napi::Result<crate::job::Job> {
let job = self.get_inner()?.open_job(&job_id).await.default_error()?;
Ok(crate::job::Job::new(job))
}
@@ -459,17 +493,6 @@ impl Connection {
Ok(jobs.into_iter().map(Into::into).collect())
}
/// Describe a single server-side job by id. `null` when the server has
/// no such job.
#[napi(catch_unwind)]
pub async fn get_job(
&self,
job_id: String,
) -> napi::Result<Option<crate::job::JobDescription>> {
let description = self.get_inner()?.get_job(&job_id).await.default_error()?;
Ok(description.map(Into::into))
}
/// Request cancellation of a server-side job by id. Returns true if the
/// server accepted the cancellation, false if no such job exists.
#[napi(catch_unwind)]
@@ -477,34 +500,6 @@ impl Connection {
self.get_inner()?.cancel_job(&job_id).await.default_error()
}
/// The lifecycle event history of a server-side job (all jobs when
/// `job_id` is null), as an Arrow IPC stream buffer. Empty when there is
/// no history.
#[napi(catch_unwind)]
pub async fn job_history(&self, job_id: Option<String>) -> napi::Result<Buffer> {
let batches = self
.get_inner()?
.job_history(job_id.as_deref())
.await
.default_error()?;
let Some(first) = batches.first() else {
return Ok(Buffer::from(Vec::<u8>::new()));
};
let mut out = Vec::new();
let mut writer = arrow_ipc::writer::StreamWriter::try_new(&mut out, &first.schema())
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
for batch in &batches {
writer
.write(batch)
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
}
writer
.finish()
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
drop(writer);
Ok(Buffer::from(out))
}
#[napi(catch_unwind)]
/// Describe a namespace and return its properties.
pub async fn describe_namespace(
nodejs/src/job.rs
+90 -34
View File
@@ -3,6 +3,9 @@
use std::sync::Arc;
use arrow_array::RecordBatch;
use lancedb::job::JobEventsRequest;
use napi::bindgen_prelude::Buffer;
use napi_derive::napi;
use crate::error::NapiErrorExt;
@@ -55,12 +58,98 @@ impl Job {
pub async fn cancel(&self) -> napi::Result<()> {
self.inner.cancel().await.default_error()
}
/// Ask the backend for this job's current state, and for a server-side job
/// its full record, then cache it for the getters below.
///
/// They are all null until this runs, because submitting an operation
/// returns only a job id. {@link Job.status} fetches the whole record too;
/// {@link Job.wait} records only the terminal state it establishes.
#[napi(catch_unwind)]
pub async fn refresh(&self) -> napi::Result<()> {
self.inner.refresh().await.default_error()
}
/// The last observed lifecycle state, without contacting the backend.
#[napi(getter)]
pub fn state(&self) -> Option<String> {
self.inner.state()
}
/// The job's type, as the server names it. Null for an in-process job,
/// which has no server-side record.
#[napi(getter)]
pub fn job_type(&self) -> Option<String> {
self.inner.job_type()
}
/// When the job was created, in milliseconds since the epoch.
#[napi(getter)]
pub fn creation_ms(&self) -> Option<i64> {
self.inner.creation_ms()
}
/// The job-type-specific specification as a JSON string, when present.
#[napi(getter)]
pub fn spec_json(&self) -> Option<String> {
self.inner.spec().map(|spec| spec.to_string())
}
/// The job-type-specific terminal result as a JSON string. Null until the
/// job succeeds, so a job that never terminates reports its progress
/// through {@link Job.events} instead.
#[napi(getter)]
pub fn result_json(&self) -> Option<String> {
self.inner.result().map(|result| result.to_string())
}
/// Why the job failed, when it failed and the server reports a reason.
#[napi(getter)]
pub fn failure(&self) -> Option<JobFailureInfo> {
self.inner.failure().map(|failure| JobFailureInfo {
phase: failure.phase,
message: failure.message,
retryable: failure.retryable,
})
}
/// This job's recorded lifecycle events, as an Arrow IPC stream buffer.
/// The TypeScript wrapper turns it into an Arrow table.
#[napi(catch_unwind)]
pub async fn events(&self, limit: Option<u32>, filter: Option<String>) -> napi::Result<Buffer> {
let batches = self
.inner
.events(JobEventsRequest { limit, filter })
.await
.default_error()?;
batches_to_ipc_buffer(&batches)
}
}
/// Serialise Arrow batches as a single IPC stream for the TypeScript layer.
fn batches_to_ipc_buffer(batches: &[RecordBatch]) -> napi::Result<Buffer> {
let Some(first) = batches.first() else {
return Ok(Buffer::from(Vec::<u8>::new()));
};
let mut out = Vec::new();
let mut writer = arrow_ipc::writer::StreamWriter::try_new(&mut out, &first.schema())
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
for batch in batches {
writer
.write(batch)
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
}
writer
.finish()
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
drop(writer);
Ok(Buffer::from(out))
}
/// A row from `Connection.listJobs`: one server-side job.
#[napi(object)]
pub struct JobInfo {
/// The job id -- what `Connection.getJob` and `Connection.cancelJob`
/// The job id -- what `Connection.openJob` and `Connection.cancelJob`
/// accept.
pub job_id: String,
/// The table the job runs against, without URI or namespace.
@@ -91,36 +180,3 @@ pub struct JobFailureInfo {
pub message: Option<String>,
pub retryable: Option<bool>,
}
/// A described job from `Connection.getJob`.
#[napi(object)]
pub struct JobDescription {
pub job_id: String,
pub job_type: String,
/// Lifecycle state: "running", "finished", "failed", or "cancelled".
pub state: String,
/// When the job was created, in milliseconds since the epoch.
pub creation_ms: i64,
/// The job-type-specific specification as a JSON string, when present.
pub spec_json: Option<String>,
/// Why the job failed, when the job is failed and the server reports a
/// reason.
pub failure: Option<JobFailureInfo>,
}
impl From<lancedb::database::JobDescription> for JobDescription {
fn from(description: lancedb::database::JobDescription) -> Self {
Self {
job_id: description.job_id,
job_type: description.job_type,
state: description.state,
creation_ms: description.creation_ms,
spec_json: (!description.spec.is_null()).then(|| description.spec.to_string()),
failure: description.failure.map(|failure| JobFailureInfo {
phase: failure.phase,
message: failure.message,
retryable: failure.retryable,
}),
}
}
}
nodejs/src/lib.rs
+2
View File
@@ -10,6 +10,8 @@ use std::collections::HashMap;
use env_logger::Env;
use napi_derive::*;
mod blob;
mod catalog;
mod connection;
mod error;
mod header;

Some files were not shown because too many files have changed in this diff Show More