rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-13 15:22:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge lancedb main a68c7be2 into the docs candidate

Browse Source
This commit is contained in:
Brendan Clement
2026-10-05 14:04:47 -07:00
parent 41ec678699 a68c7be202
commit 07fc8f46d6
279 changed files with 56002 additions and 4845 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.bumpversion.toml
+1 -1
View File
@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.39.0-beta.1"
current_version = "0.40.0-beta.12"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.
.github/release_notes.json
+5 -1
View File
@@ -6,6 +6,10 @@
"title": "## 🏆 Highlights",
"labels": ["highlight"]
},
{
"title": "## 🚨 Critical Fixes",
"labels": ["critical-fix"]
},
{
"title": "## 🛠 Breaking Changes",
"labels": ["breaking-change"]
@@ -38,4 +42,4 @@
"labels": ["ci"]
}
]
}
}
.github/workflows/nodejs.yml
+66 -1
View File
@@ -164,6 +164,71 @@ jobs:
python ci/mock_openai.py &
cd nodejs/examples
node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose
arrow-compat:
# The main jobs resolve `apache-arrow` to the version pinned in
# devDependencies (the newest supported release). This job re-runs the
# suite with the library itself running on each older Arrow release the
# peer range still accepts, which is what a consumer on that version gets.
name: Linux (Arrow ${{ matrix.arrow-version }})
timeout-minutes: 30
strategy:
matrix:
arrow-version: [ "19.0.1", "20.0.0" ]
runs-on: "ubuntu-22.04"
defaults:
run:
shell: bash
working-directory: nodejs
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
lfs: true
- uses: pnpm/action-setup@v6
with:
version: 11.1.1
- uses: actions/setup-node@v6
with:
node-version: 24
cache: 'pnpm'
cache-dependency-path: nodejs/pnpm-lock.yaml
- uses: Swatinem/rust-cache@v2
with:
# Same crate, profile and target as the `linux` job above, so reuse
# its cache: rust-cache keys on the job id by default, and this job
# would otherwise start cold on every run until it had saved its own.
shared-key: linux
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Install dependencies
run: |
sudo apt update
sudo apt install -y protobuf-compiler libssl-dev
- name: Build
run: |
pnpm install --frozen-lockfile
# No `--` separator: pnpm forwards it literally, which would
# make napi-rs treat `--profile ci` as a cargo passthrough arg.
pnpm build:debug --profile ci
- name: Resolve apache-arrow to ${{ matrix.arrow-version }}
run: |
# The older releases are already installed as the `apache-arrow-NN`
# test aliases. With pnpm's hoisted linker each alias is a complete,
# self-contained directory, so swap it in for the library's own
# `apache-arrow` instead of running a second install, which has been
# seen to hang after printing "Done" in CI. A copy, not a symlink: the
# cross-version tests rely on the alias being a separate module
# instance from the one the library loads.
major="$(echo '${{ matrix.arrow-version }}' | cut -d. -f1)"
mv node_modules/apache-arrow node_modules/apache-arrow.pinned
cp -R "node_modules/apache-arrow-${major}" node_modules/apache-arrow
# `require("apache-arrow/package.json")` is blocked by the package's exports map.
node -p "'apache-arrow resolved to ' + JSON.parse(require('fs').readFileSync('node_modules/apache-arrow/package.json')).version"
pnpm tsc
- name: Test
env:
# Newer @smithy/core uses dynamic ESM imports.
NODE_OPTIONS: "--experimental-vm-modules"
run: node_modules/.bin/jest --verbose
macos:
timeout-minutes: 30
# macos-15 ships a newer linker; the older macos-14 linker fails to insert
@@ -207,4 +272,4 @@ jobs:
pnpm tsc
- name: Test
run: |
pnpm test
pnpm test --runInBand
.github/workflows/pypi-publish.yml
+9 -10
View File
@@ -21,7 +21,7 @@ permissions:
contents: read
# Without this, a force-push to a PR leaves the previous run going -- including
# a ~74 minute Windows job and a billed arm64 wheel build.
# a long fat-LTO Windows job and a billed arm64 wheel build.
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
@@ -123,18 +123,17 @@ jobs:
path: target/wheels/lancedb-*.whl
if-no-files-found: error
windows:
timeout-minutes: 90
runs-on: windows-latest
# The fat-LTO build of the `lancedb` crate does not finish within 90 minutes
# on the 4-core standard runner, so use the 8-core larger runner with extra
# headroom.
timeout-minutes: 150
runs-on: windows-2025-8x-x64
env:
# link.exe is single-threaded and the long pole on Windows builds. Use
# rustc's bundled lld-link instead.
CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER: rust-lld
# Fat LTO of the cdylib is single-threaded and the peak-memory step of the
# build. ThinLTO parallelizes it across the runner's cores, at some cost
# to runtime performance on our least performance-sensitive platform.
# Matches what the nodejs Windows builds already do in npm-publish.yml.
CARGO_PROFILE_RELEASE_LTO: thin
CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 16
# Keep the repository's fat LTO / 1 codegen unit release profile to leave
# headroom below PyPI's file-size limit for Windows wheels.
steps:
- uses: actions/checkout@v6
with:
@@ -149,7 +148,7 @@ jobs:
# or the default branch -- so with no run on main there is nothing that
# can populate an entry the release build would be allowed to read. Fixing
# this needs a main/nightly trigger (which would also catch wheel-build
# breakage before a release); the ~74 minutes here is otherwise dominated
# breakage before a release); the build time here is otherwise dominated
# by the fat-LTO link, which no cache avoids.
- uses: ./.github/workflows/build_windows_wheel
with:
.github/workflows/python.yml
+1 -1
View File
@@ -156,7 +156,7 @@ jobs:
- name: Test without pylance or pandas
run: |
pip uninstall -y pylance pandas
pytest -vv python/tests/test_table.py
pytest -vv python/tests/test_table.py python/tests/test_namespace_no_pylance.py
# Make sure wheels are not included in the Rust cache
- name: Delete wheels
run: rm -rf target/wheels
.github/workflows/rust.yml
+6 -2
View File
@@ -180,7 +180,8 @@ jobs:
env:
CC: clang-18
CXX: clang++-18
GH_TOKEN: ${{ secrets.SOPHON_READ_TOKEN }}
GH_TOKEN: ${{ github.token }}
SOPHON_READ_TOKEN: ${{ secrets.SOPHON_READ_TOKEN }}
steps:
- uses: actions/checkout@v6
with:
@@ -232,7 +233,10 @@ jobs:
ALL_FEATURES=`cargo metadata --format-version=1 --no-deps \
| jq -r '.packages[] | .features | keys | .[]' \
| grep -v s3-test | sort | uniq | paste -s -d "," -`
cargo test --profile ci --features $ALL_FEATURES --locked
# Run doctests before test binaries fill the runner disk. Examples are
# already built by the Linux job, so avoid retaining them here.
cargo test --profile ci --features $ALL_FEATURES --locked --doc
cargo test --profile ci --features $ALL_FEATURES --locked --lib --tests
windows:
strategy:
.github/workflows/typos.yml
+20
View File
@@ -0,0 +1,20 @@
name: Typo checker
on:
push:
branches:
- main
pull_request:
permissions:
contents: read
jobs:
run:
name: Spell Check with Typos
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Check spelling of the entire repository
uses: crate-ci/typos@6802cc60d4e7f78b9d5454f6cf3935c042d5e1e3 # v1.26.0
.pre-commit-config.yaml
+4
View File
@@ -10,6 +10,10 @@ repos:
rev: v0.9.9
hooks:
- id: ruff
- repo: https://github.com/crate-ci/typos
rev: v1.26.0
hooks:
- id: typos
# - repo: https://github.com/RobertCraigie/pyright-python
# rev: v1.1.395
# hooks:
.typos.toml
+19
View File
@@ -0,0 +1,19 @@
[default]
extend-ignore-re = ["(?Rm)^.*(#|//)\\s*spellchecker:disable-line$"]
[default.extend-words]
# Azure Kubernetes Service, mentioned in rust/lancedb/src/remote/oauth.rs.
AKS = "AKS"
# RabitQ is the name of a vector quantization algorithm, not a typo of "Rabbit".
Rabit = "Rabit"
# `VarBuilder::from_mmaped_safetensors` is the real (if oddly-spelled) name of
# the candle-core API we call in rust/lancedb/src/embeddings/sentence_transformers.rs.
mmaped = "mmaped"
# `WriteableBuffer` is the real name of a type from Python's `_typeshed` stubs,
# used in python/python/lancedb/_blob.py.
Writeable = "Writeable"
[files]
extend-exclude = [
"*_THIRD_PARTY_LICENSES.*",
]
CLAUDE.md
-1
View File
@@ -1 +0,0 @@
AGENTS.md
Cargo.lock
Generated
+891 -309
View File
File diff suppressed because it is too large Load Diff
Cargo.toml
+21 -16
View File
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
rust-version = "1.91.0"
[workspace.dependencies]
lance = { "version" = "=12.0.0-beta.11", default-features = false, "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=12.0.0-beta.11", default-features = false, "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=12.0.0-beta.11", default-features = false, "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=13.1.0-beta.1", default-features = false, "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=13.1.0-beta.1", default-features = false, "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=13.1.0-beta.1", default-features = false, "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lancedb = { path = "rust/lancedb", default-features = false }
ahash = "0.8"
# Note that this one does not include pyarrow
@@ -39,7 +39,10 @@ arrow-ord = "58.0.0"
arrow-schema = "58.0.0"
arrow-select = "58.0.0"
arrow-cast = "58.0.0"
arrow-flight = { version = "58.0.0", features = ["flight-sql-experimental"] }
async-trait = "0"
# Smithy JSON 0.63 requires the pre-1.7 Document representation; allow the MSRV pin.
aws-smithy-types = ">=1.3.6, <1.7"
bytes = "1"
datafusion = { version = "54.0.0", default-features = false }
datafusion-catalog = "54.0.0"
@@ -59,19 +62,21 @@ log = "0.4"
metrics = "0.24"
metrics-util = "0.19"
moka = { version = "0.12", features = ["future"] }
object_store = "0.13.2"
object_store = "0.14.1"
pin-project = "1.0.7"
rand = "0.9"
snafu = "0.8"
url = "2"
num-traits = "0.2"
oauth2 = { version = "5.0", default-features = false }
regex = "1.10"
semver = "1.0.25"
serde = "1"
serde_json = "1"
tempfile = "3.5.0"
tokio = { version = "1.23", features = ["rt-multi-thread", "sync"] }
uuid = { version = "1.7.0", features = ["v4"] }
tonic = { version = "0.14", features = ["tls-native-roots", "tls-ring"] }
uuid = { version = "1.7.0", features = ["v4", "v7"] }
chrono = { version = "0.4", default-features = false, features = ["clock"] }
[profile.ci]
docs/openapi.yml
+1 -1
View File
@@ -155,7 +155,7 @@ paths:
vector:
type: FixedSizeList
description: |
The targetted vector to search for. Required.
The targeted vector to search for. Required.
vector_column:
type: string
description: |
docs/src/java/java.md
+1 -1
View File
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
<dependency>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-core</artifactId>
<version>0.39.0-beta.1</version>
<version>0.40.0-beta.12</version>
</dependency>
```
docs/src/js/classes/AutoQuery.md
+3 -1
View File
@@ -350,7 +350,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
@@ -417,6 +417,8 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
#### Inherited from
`StandardQueryBase.toArrow`
docs/src/js/classes/BlobFile.md
+158
View File
@@ -0,0 +1,158 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobFile
# Class: BlobFile
A lazy handle to blob bytes. Create one with [Table.fetchBlobFiles](Table.md#fetchblobfiles).
## Methods
### close()
```ts
close(): Promise<void>
```
Releases the handle. Reads after `close()` fail. Calling it again does
nothing.
#### Returns
`Promise`&lt;`void`&gt;
***
### isClosed()
```ts
isClosed(): boolean
```
Returns true after [BlobFile.close](BlobFile.md#close).
#### Returns
`boolean`
***
### read()
```ts
read(maxBytes?, options?): Promise<Buffer>
```
Reads from the cursor and advances the cursor.
Reads to the end when `maxBytes` is omitted, or at most `maxBytes` bytes
otherwise. Returns an empty buffer at the end of the blob.
[BlobFile.readRange](BlobFile.md#readrange) does not move the cursor.
#### Parameters
* **maxBytes?**: `bigint`
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;`Buffer`&gt;
***
### readRange()
```ts
readRange(
start,
end,
options?): Promise<Buffer>
```
Reads the half-open byte range `[start, end)`.
Fails when `end` is past the blob size. Does not move the cursor.
#### Parameters
* **start**: `bigint`
* **end**: `bigint`
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;`Buffer`&gt;
***
### readRanges()
```ts
readRanges(ranges, options?): Promise<Buffer[]>
```
Reads several half-open byte ranges. Returns one buffer per range, in
the order given.
Fails when any `end` is past the blob size. Does not move the cursor.
#### Parameters
* **ranges**: [`BlobRange`](../type-aliases/BlobRange.md)[]
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;`Buffer`[]&gt;
***
### seek()
```ts
seek(position): Promise<void>
```
Moves the cursor to `position`, in bytes from the start of the blob.
#### Parameters
* **position**: `bigint`
#### Returns
`Promise`&lt;`void`&gt;
***
### size()
```ts
size(): bigint
```
Returns the blob size in bytes.
#### Returns
`bigint`
***
### tell()
```ts
tell(): Promise<bigint>
```
Returns the cursor position in bytes.
#### Returns
`Promise`&lt;`bigint`&gt;
docs/src/js/classes/Catalog.md
+107
View File
@@ -0,0 +1,107 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / Catalog
# Class: Catalog
A remote catalog manages databases through the server's root namespace.
## Accessors
### uri
```ts
get uri(): string
```
The root namespace endpoint.
#### Returns
`string`
## Methods
### connectDatabase()
```ts
connectDatabase(name): Promise<Connection>
```
Connect to an existing database by its logical name.
#### Parameters
* **name**: `string`
#### Returns
`Promise`&lt;[`Connection`](Connection.md)&gt;
***
### createDatabase()
```ts
createDatabase(name, options): Promise<Connection>
```
Create a database, or open an existing database when existOk is true.
#### Parameters
* **name**: `string`
* **options** = `{}`
* **options.existOk?**: `boolean`
#### Returns
`Promise`&lt;[`Connection`](Connection.md)&gt;
***
### dropDatabase()
```ts
dropDatabase(name, options): Promise<void>
```
Drop an empty database. The server rejects nonempty databases.
#### Parameters
* **name**: `string`
* **options** = `{}`
* **options.ignoreMissing?**: `boolean`
#### Returns
`Promise`&lt;`void`&gt;
***
### listDatabases()
```ts
listDatabases(options): Promise<ListDatabasesResponse>
```
List one page of databases; pass pageToken from a response for the next page.
#### Parameters
* **options** = `{}`
* **options.limit?**: `number`
* **options.pageToken?**: `string`
#### Returns
`Promise`&lt;[`ListDatabasesResponse`](../interfaces/ListDatabasesResponse.md)&gt;
docs/src/js/classes/Connection.md
+229 -53
View File
@@ -180,13 +180,13 @@ abstract createMaterializedView(
Define a materialized view named `name` over the table `source`.
The view is created empty, with the query recorded in its schema
metadata; `view.refresh()` computes the rows. The view is a normal
table: it can be queried, indexed and searched, and it appears in
`tableNames`. The source table must have stable row ids (create it with
The view is populated before creation returns. Set `withNoData` to create
only its definition and empty backing table. The view is a normal table:
it can be queried, indexed and searched, and it appears in `tableNames`.
The source table must have stable row ids (create it with
the `newTableEnableStableRowIds` storage option); they keep the view's
provenance valid across source compactions and cannot be enabled after
a table exists. Local databases only.
a table exists.
#### Parameters
@@ -202,6 +202,8 @@ a table exists. Local databases only.
* **options.where?**: `string`
* **options.withNoData?**: `boolean`
#### Returns
`Promise`&lt;[`MaterializedView`](MaterializedView.md)&gt;
@@ -317,6 +319,38 @@ Creates a new Table and initialize it with new data.
***
### createView()
```ts
abstract createView(
name,
query,
namespacePath?): Promise<ViewDescription>
```
Create a view: a named query the database plans on every read.
The query is planned once, at creation, so one that cannot be planned is
rejected now rather than at the first read. A view holds no rows, and its
readers see its sources as they are at read time.
There is no replace: a name already taken is an error, and changing a
view is a drop followed by a create.
#### Parameters
* **name**: `string`
* **query**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;[`ViewDescription`](../interfaces/ViewDescription.md)&gt;
***
### describeNamespace()
```ts
@@ -340,6 +374,27 @@ The namespace's properties
***
### describeView()
```ts
abstract describeView(name, namespacePath?): Promise<ViewDescription>
```
What this database records about the view named `name`: its defining
query and the schema that query resolved to.
#### Parameters
* **name**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;[`ViewDescription`](../interfaces/ViewDescription.md)&gt;
***
### display()
```ts
@@ -373,6 +428,54 @@ Drop all tables in the database.
***
### dropMaterializedView()
```ts
abstract dropMaterializedView(name, namespacePath?): Promise<void>
```
Drop the materialized view named `name`.
The view may become unavailable before physical cleanup finishes. Use
[dropMaterializedViewAsync](Connection.md#dropmaterializedviewasync) to retain and wait for the cleanup job.
Rejects a table that exists but is not a materialized view.
#### Parameters
* **name**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;`void`&gt;
***
### dropMaterializedViewAsync()
```ts
abstract dropMaterializedViewAsync(name, namespacePath?): Promise<Job>
```
Start dropping the materialized view named `name` and return its cleanup
job without waiting for completion.
Rejects a table that exists but is not a materialized view.
#### Parameters
* **name**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;[`Job`](Job.md)&gt;
***
### dropNamespace()
```ts
@@ -448,23 +551,50 @@ on the returned job to know when cleanup has finished.
***
### getJob()
### dropView()
```ts
abstract getJob(jobId): Promise<null | JobDescription>
abstract dropView(name, namespacePath?): Promise<void>
```
Describe a single server-side job by id.
Drop the view named `name` and wait for its definition to be deleted.
Resolves to `null` when the server has no such job.
The tables it reads are untouched: a view holds no rows of its own. Use
[dropViewAsync](Connection.md#dropviewasync) to retain the cleanup job instead of waiting on it.
#### Parameters
* **jobId**: `string`
* **name**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;`null` \| [`JobDescription`](../interfaces/JobDescription.md)&gt;
`Promise`&lt;`void`&gt;
***
### dropViewAsync()
```ts
abstract dropViewAsync(name, namespacePath?): Promise<Job>
```
Start dropping the view named `name` and return the job deleting its
definition, without waiting for completion.
The name is free before this resolves. When nothing was bound to it, the
returned job is already finished and has no id.
#### Parameters
* **name**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;[`Job`](Job.md)&gt;
***
@@ -482,48 +612,6 @@ Return true if the connection has not been closed
***
### job()
```ts
abstract job(jobId): Job
```
A [Job](Job.md) handle for a server-side job by id.
The handle is constructed without a server round trip; an unknown id
surfaces when the handle is used. Dropping the handle has no effect on
the job itself.
#### Parameters
* **jobId**: `string`
#### Returns
[`Job`](Job.md)
***
### jobHistory()
```ts
abstract jobHistory(jobId?): Promise<Table<any>>
```
The lifecycle event history of a server-side job, as an Arrow table.
Lists history across all jobs when `jobId` is omitted.
#### Parameters
* **jobId?**: `string`
#### Returns
`Promise`&lt;`Table`&lt;`any`&gt;&gt;
***
### listJobs()
```ts
@@ -648,6 +736,50 @@ A page of table names and an
***
### listViews()
```ts
abstract listViews(namespacePath?): Promise<string[]>
```
The names of the views in one namespace.
Names only; a definition comes from [describeView](Connection.md#describeview).
#### Parameters
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;`string`[]&gt;
***
### openJob()
```ts
abstract openJob(jobId): Promise<Job>
```
Open a server-side job by id, returning a handle with its record already
populated. Rejects when the server has no such job, the way
[Connection.openTable](Connection.md#opentable) does for a missing table.
The returned [Job](Job.md) answers for its own state, specification,
result, failure and event history, so there is no separate
connection-level call for any of them.
#### Parameters
* **jobId**: `string`
#### Returns
`Promise`&lt;[`Job`](Job.md)&gt;
***
### openMaterializedView()
```ts
@@ -691,6 +823,28 @@ abstract openTable(
***
### pauseJob()
```ts
abstract pauseJob(jobId): Promise<string>
```
Pause a server-side job by id.
The job's workers drain and it stays parked until resumed. Resolves to
"pausing", "already_paused", or "committing" -- a job finalizing its
results cannot be parked; retry shortly.
#### Parameters
* **jobId**: `string`
#### Returns
`Promise`&lt;`string`&gt;
***
### renameTable()
```ts
@@ -724,6 +878,28 @@ a "not supported" error.
***
### resumeJob()
```ts
abstract resumeJob(jobId): Promise<string>
```
Resume a paused server-side job by id.
Its workers pick their work back up from checkpoints. Resolves to
"resumed", "still_pausing" -- the pause's worker drain is not confirmed
yet; retry shortly -- or "not_paused".
#### Parameters
* **jobId**: `string`
#### Returns
`Promise`&lt;`string`&gt;
***
### ~~tableNames()~~
#### tableNames(options)
docs/src/js/classes/Job.md
+159 -12
View File
@@ -8,19 +8,46 @@
A handle to an operation that may still be running.
## Constructors
The operation may already be complete when the handle is created.
### new Job()
The detail getters read what the handle last observed. Submitting an
operation returns only a job id, so populating them eagerly would cost an
extra round trip on every call:
- [Job.refresh](Job.md#refresh) and [Job.status](Job.md#status) fetch the whole record.
- [Job.wait](Job.md#wait) records the terminal state it establishes, but not the
rest of the record.
- Everything is null until one of those runs.
## Accessors
### creationMs
```ts
new Job(): Job
get creationMs(): null | number
```
When the job was created, in milliseconds since the epoch.
#### Returns
[`Job`](Job.md)
`null` \| `number`
## Accessors
***
### failure
```ts
get failure(): null | JobFailureInfo
```
Why the job failed, when it failed and the server reports a reason.
#### Returns
`null` \| [`JobFailureInfo`](../interfaces/JobFailureInfo.md)
***
### id
@@ -28,8 +55,69 @@ new Job(): Job
get id(): null | string
```
Identifies the operation on the server that is running it. Operations
that run in this process have no server id. The value is opaque.
Identifies the operation on the server that is running it.
Operations that run in this process have no server id. The value is
opaque: parsing it or storing it to resume the job later is not supported.
#### Returns
`null` \| `string`
***
### jobType
```ts
get jobType(): null | string
```
The job's type, as the server names it. Null for an in-process job, which
has no server-side record.
#### Returns
`null` \| `string`
***
### result
```ts
get result(): any
```
The job-type-specific terminal result. Null until the job succeeds, so a
job that never terminates reports its progress through [Job.events](Job.md#events)
instead.
#### Returns
`any`
***
### spec
```ts
get spec(): any
```
The job-type-specific specification it was submitted with.
#### Returns
`any`
***
### state
```ts
get state(): null | string
```
The last observed lifecycle state, without contacting the backend.
#### Returns
@@ -51,18 +139,61 @@ Request cancellation. Cancelling a finished operation is a no-op.
***
### events()
```ts
events(options?): Promise<Table<any>>
```
This job's recorded lifecycle events.
Where the getters above report a terminal result only once the job reaches
one, events are written as the job runs and outlive the workers that
produced them. A distributed job records a `claim`/`claim_complete` pair
per unit of work, each carrying `rows_processed`, so a job that never
finishes still accounts for what it did.
The server caps results at 1000 rows by default and 10,000 at most, and
truncates without saying so, so pass `limit` for a job that emits an event
per fragment. `filter` is a SQL-like expression over the `state`,
`updated_by`, `emitted_from`, `emitted_by`, and `claim_entity` columns.
#### Parameters
* **options?**: [`JobEventsOptions`](../interfaces/JobEventsOptions.md)
#### Returns
`Promise`&lt;`Table`&lt;`any`&gt;&gt;
***
### refresh()
```ts
refresh(): Promise<void>
```
Ask the backend for this job's current state, and for a server-side job
its full record, then cache it for the getters above.
#### Returns
`Promise`&lt;`void`&gt;
***
### status()
```ts
status(): Promise<string>
```
The operation's current lifecycle state: "running", "finished",
"failed", or "cancelled".
The operation's current lifecycle state: "running", "finished", "failed",
or "cancelled".
A point snapshot; unlike [Job.wait](Job.md#wait) it does not block or reject
on a terminal failure state. States a newer server reports that this
client version does not know pass through as-is.
A point snapshot; unlike [Job.wait](Job.md#wait) it does not block or reject on a
terminal failure state. Also refreshes the getters above.
#### Returns
@@ -70,6 +201,22 @@ client version does not know pass through as-is.
***
### toString()
```ts
toString(): string
```
Every field the handle currently knows, one per line, with the JSON
payloads indented -- a refresh job's spec and result are the point of
printing it.
#### Returns
`string`
***
### wait()
```ts
docs/src/js/classes/MaterializedView.md
+1 -1
View File
@@ -49,7 +49,7 @@ get name(): string
definition(): Promise<MaterializedViewDefinition>
```
The query that defines the view, read from its stored schema.
The query that defines the view.
#### Returns
docs/src/js/classes/MergeInsertBuilder.md
+1 -1
View File
@@ -141,7 +141,7 @@ Currently this causes multiple copies of the row to be created
but that behavior is subject to change.
An optional condition may be specified. If it is, then only
matched rows that satisfy the condtion will be updated. Any
matched rows that satisfy the condition will be updated. Any
rows that do not satisfy the condition will be left as they
are. Failing to satisfy the condition does not cause a
"matched row" to become a "not matched" row.
docs/src/js/classes/OAuthSession.md
+105
View File
@@ -0,0 +1,105 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / OAuthSession
# Class: OAuthSession
Explicit OAuth session lifecycle for the persistent token cache: eager
`login`, non-secret `status`, and local `logout`.
A session is built from the same [OAuthConfig](../interfaces/OAuthConfig.md) used to connect
(including its `tokenCache` options). A connection created with the same
configuration shares the cache, so logging in here prepares tokens for
later processes without any database request.
`login` always runs the configured interactive flow and replaces the cached
session (the most recent login wins). `logout` removes only the local
credential; it does not revoke anything with the provider and does not sign
out of a browser SSO session.
## Example
```typescript
const config: OAuthConfig = {
issuerUrl: "https://issuer.example.com",
clientId: "my-app",
scopes: ["openid", "offline_access"],
flow: OAuthFlowType.DeviceCode,
tokenCache: { cacheDir: "/tmp/my-app/oauth-cache" },
};
const session = new OAuthSession(config);
const status = await session.login();
```
## Constructors
### new OAuthSession()
```ts
new OAuthSession(config): OAuthSession
```
Create a session manager for the given OAuth configuration.
#### Parameters
* **config**: [`OAuthConfig`](../interfaces/OAuthConfig.md)
#### Returns
[`OAuthSession`](OAuthSession.md)
## Methods
### login()
```ts
login(): Promise<SessionStatus>
```
Eagerly run the configured authentication flow and store the session.
A successful login always replaces any prior cached session for this
identity; if the provider does not issue a refresh token (for example
without `offline_access`), the previous record is removed and the status
reports `refreshable == false`.
#### Returns
`Promise`&lt;[`SessionStatus`](../interfaces/SessionStatus.md)&gt;
***
### logout()
```ts
logout(): Promise<SessionLogout>
```
Remove the matching local cached credential.
This only deletes the local cache entry. It does not revoke the refresh
token with the provider and does not sign out of a browser SSO session.
Repeated calls succeed; `removed` reports whether a credential existed.
#### Returns
`Promise`&lt;[`SessionLogout`](../interfaces/SessionLogout.md)&gt;
***
### status()
```ts
status(): Promise<SessionStatus>
```
Report whether a matching cached session exists, with safe metadata.
This never contacts the identity provider and never exposes token values.
#### Returns
`Promise`&lt;[`SessionStatus`](../interfaces/SessionStatus.md)&gt;
docs/src/js/classes/Query.md
+3 -1
View File
@@ -424,7 +424,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
@@ -491,6 +491,8 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
#### Inherited from
`StandardQueryBase.toArrow`
docs/src/js/classes/QueryBase.md
+3 -1
View File
@@ -193,7 +193,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
@@ -252,6 +252,8 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
***
### withRowId()
docs/src/js/classes/Table.md
+121 -10
View File
@@ -74,10 +74,10 @@ now: the column is committed with no values, and rows get them from
[Table#refreshColumn](Table.md#refreshcolumn). Declaring one therefore costs the same on a
large table as on an empty one.
A refresh does not revisit rows it has already filled, so mutating an
input leaves the value computed at fill time; recomputing means dropping
the column and declaring it again. While a declaration reads a column,
that column cannot be renamed, retyped or dropped.
A refresh also recomputes the rows whose inputs changed since they were
computed, so a mutated input is reflected by the next refresh. While a
declaration reads a column, that column cannot be renamed, retyped or
dropped.
On LanceDB Cloud and Enterprise the expression is planned by the
server, and the refresh runs as a server job -- see
@@ -137,6 +137,20 @@ containing the new version number of the table after altering the columns.
***
### blobColumns()
```ts
abstract blobColumns(): Promise<string[]>
```
Blob v2 columns, including nested dotted paths.
#### Returns
`Promise`&lt;`string`[]&gt;
***
### branches()
```ts
@@ -499,6 +513,68 @@ Drop an index from the table.
***
### fetchBlobFiles()
```ts
abstract fetchBlobFiles(
column,
rowIds,
options?): Promise<(null | BlobFile)[]>
```
Opens lazy blob handles for `column` at the given row IDs using the
table's current checkout.
Preserves input order, duplicates, and nulls. Use this for large payloads.
See [Table.fetchBlobs](Table.md#fetchblobs) for row-ID validity across versions.
#### Parameters
* **column**: `string`
* **rowIds**: readonly (`number` \| `bigint`)[]
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;(`null` \| [`BlobFile`](BlobFile.md))[]&gt;
***
### fetchBlobs()
```ts
abstract fetchBlobs(
column,
rowIds,
options?): Promise<(null | Buffer)[]>
```
Bytes for `column` at row IDs from [Query.withRowId](Query.md#withrowid).
Reads the table's current checkout. IDs from another version can fail after
compaction unless stable row ids are enabled. Results keep input order and
duplicates. Null blobs are `null`. Empty blobs are empty buffers.
Remote servers limit each request to 1024 row IDs and 64 MiB of blob bytes.
The client splits requests automatically and reads an individual larger
blob through the Range route. This method still materializes all bytes in
memory; use [Table.fetchBlobFiles](Table.md#fetchblobfiles) for large values.
#### Parameters
* **column**: `string`
* **rowIds**: readonly (`number` \| `bigint`)[]
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;(`null` \| `Buffer`)[]&gt;
***
### flushLsm()
```ts
@@ -516,6 +592,41 @@ so this is safe to call repeatedly.
***
### functionErrors()
```ts
abstract functionErrors(options?): Promise<FunctionErrors>
```
The per-row errors Function refreshes recorded on this table.
A refresh running under a skip policy records each row it skipped with
the input that failed and the error. This lists those records, newest
job first, plus a summary for any fragment whose per-row detail was
capped. LanceDB Cloud and Enterprise only; reading errors needs read
access to the table, since a message carries the value that failed.
#### Parameters
* **options?**: [`FunctionErrorsOptions`](../interfaces/FunctionErrorsOptions.md)
Optional filters: `jobId`,
`column`, and `limit` (server default 10000, cap 100000).
#### Returns
`Promise`&lt;[`FunctionErrors`](../interfaces/FunctionErrors.md)&gt;
The records, the capped fragments,
and whether the listing stopped at its limit.
#### Example
```ts
const { records, truncated } = await table.functionErrors({ column: "embedding" });
```
***
### getLsmStats()
```ts
@@ -854,10 +965,10 @@ abstract refreshColumn(column): Promise<RefreshColumnResult>
Fill the rows of a computed column that hold no value yet.
Rows appended since the last refresh are filled by the next one; rows
already filled are left as they are, so the call is idempotent and does
not observe a mutated input. Local tables only: a remote refresh runs
as a server job, through [Table#refreshColumnAsync](Table.md#refreshcolumnasync).
Rows appended since the last refresh are filled by the next one, and
rows whose inputs changed since they were computed are recomputed;
everything else is left as it is. Local tables only: a remote refresh
runs as a server job, through [Table#refreshColumnAsync](Table.md#refreshcolumnasync).
#### Parameters
@@ -1266,11 +1377,11 @@ value is 0")
Note: if your condition is something like "some_id_column == 7" and
you are updating many rows (with different ids) then you will get
better performance with a single [`merge_insert`] call instead of
repeatedly calilng this method.
repeatedly calling this method.
##### Parameters
* **updates**: `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **updates**: `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
the
columns to update
docs/src/js/classes/TakeQuery.md
+3 -1
View File
@@ -200,7 +200,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
@@ -267,6 +267,8 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
#### Inherited from
[`QueryBase`](QueryBase.md).[`toArrow`](QueryBase.md#toarrow)
docs/src/js/classes/VectorQuery.md
+5 -1
View File
@@ -447,6 +447,8 @@ nprobes(nprobes): VectorQuery
Set the number of partitions to search (probe)
The number of probes must be greater than 0.
This argument is only used when the vector column has an IVF PQ index.
If there is no index then this value is ignored.
@@ -673,7 +675,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
@@ -740,6 +742,8 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
#### Inherited from
`StandardQueryBase.toArrow`
docs/src/js/enumerations/ClientAuthMethod.md
+48
View File
@@ -0,0 +1,48 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / ClientAuthMethod
# Enumeration: ClientAuthMethod
How the client authenticates to the OAuth token endpoint.
The method applies to every OAuth request that carries client
authentication: client-credentials, authorization-code exchange,
refresh-token, and device-authorization requests. The Azure managed
identity flow ignores this option.
## Enumeration Members
### ClientSecretBasic
```ts
ClientSecretBasic: "client_secret_basic";
```
HTTP Basic authentication. This is the RFC 6749 recommended method and
the normal default for confidential clients, including default Okta
applications. Requires `clientSecret`.
***
### ClientSecretPost
```ts
ClientSecretPost: "client_secret_post";
```
Credentials in the request body, for providers configured to require it.
Requires `clientSecret`.
***
### None
```ts
None: "none";
```
No client authentication, for public clients using PKCE or the device
flow. Cannot be combined with `clientSecret`.
docs/src/js/enumerations/OAuthFlowType.md
+20
View File
@@ -10,6 +10,16 @@ OAuth authentication flow types.
## Enumeration Members
### AuthorizationCode
```ts
AuthorizationCode: "authorization_code";
```
Interactive Authorization Code grant, using PKCE by default.
***
### AzureManagedIdentity
```ts
@@ -27,3 +37,13 @@ ClientCredentials: "client_credentials";
```
Client Credentials grant (service-to-service / M2M).
***
### DeviceCode
```ts
DeviceCode: "device_code";
```
Device Authorization grant for CLI and headless environments.
docs/src/js/functions/blob.md
+55
View File
@@ -0,0 +1,55 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / blob
# Function: blob()
```ts
function blob(name, options): Field
```
Declares a `lance.blob.v2` column.
Query results are descriptors, not payload bytes. Use [Table.fetchBlobs](../classes/Table.md#fetchblobs)
or [Table.fetchBlobFiles](../classes/Table.md#fetchblobfiles) to read bytes.
## Parameters
* **name**: `string`
* **options**: [`BlobOptions`](../type-aliases/BlobOptions.md) = `{}`
## Returns
`Field`
## Example
```ts
import { readFile } from "node:fs/promises";
import { Field, Int64, Schema } from "apache-arrow";
import { blob, connect } from "@lancedb/lancedb";
const db = await connect("./data");
const video = await readFile("clip.mp4");
const table = await db.createTable(
"videos",
[{ id: 1n, video }],
{
schema: new Schema([
new Field("id", new Int64()),
blob("video"),
]),
},
);
const rows = await table.query().select(["id"]).withRowId().toArray();
const rowIds = rows.map((row) => row._rowid as bigint);
const bytes = await table.fetchBlobs("video", rowIds);
const [handle] = await table.fetchBlobFiles("video", rowIds);
const size = handle!.size();
const header = await handle!.readRange(0n, size < 65536n ? size : 65536n);
```
docs/src/js/functions/connectCatalog.md
+32
View File
@@ -0,0 +1,32 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / connectCatalog
# Function: connectCatalog()
```ts
function connectCatalog(endpoint, options): Promise<Catalog>
```
Connect to an HTTP(S) catalog endpoint. Catalog requests omit database-selection
headers; opened database connections inherit authentication and client options.
## Parameters
* **endpoint**: `string`
* **options**: [`CatalogOptions`](../interfaces/CatalogOptions.md) = `{}`
## Returns
`Promise`&lt;[`Catalog`](../classes/Catalog.md)&gt;
## Example
```ts
const catalog = await connectCatalog("https://my-server.example", { apiKey: "secret" });
const db = await catalog.createDatabase("analytics", { existOk: true });
const page = await catalog.listDatabases({ limit: 20 });
```
docs/src/js/functions/isBlobField.md
+22
View File
@@ -0,0 +1,22 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / isBlobField
# Function: isBlobField()
```ts
function isBlobField(field): boolean
```
Checks for the `lance.blob.v2` extension marker. Does not validate the
field's storage type.
## Parameters
* **field**: `Field`&lt;`any`&gt;
## Returns
`boolean`
docs/src/js/functions/makeJsonField.md
+36
View File
@@ -0,0 +1,36 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / makeJsonField
# Function: makeJsonField()
```ts
function makeJsonField(name, nullable): Field
```
Create an Arrow field backed by LanceDB's JSON extension type.
## Parameters
* **name**: `string`
The field name.
* **nullable**: `boolean` = `true`
Whether the field accepts null values.
## Returns
`Field`
## Example
```ts
import { connect, makeJsonField } from "@lancedb/lancedb";
import { Schema } from "apache-arrow";
const schema = new Schema([makeJsonField("metadata")]);
const db = await connect("/path/to/database");
await db.createTable("items", [{ metadata: '{"source":"api"}' }], { schema });
```
docs/src/js/globals.md
+26 -1
View File
@@ -11,6 +11,7 @@
## Enumerations
- [ClientAuthMethod](enumerations/ClientAuthMethod.md)
- [FullTextQueryType](enumerations/FullTextQueryType.md)
- [OAuthFlowType](enumerations/OAuthFlowType.md)
- [Occur](enumerations/Occur.md)
@@ -19,10 +20,12 @@
## Classes
- [AutoQuery](classes/AutoQuery.md)
- [BlobFile](classes/BlobFile.md)
- [BooleanQuery](classes/BooleanQuery.md)
- [BoostQuery](classes/BoostQuery.md)
- [BranchContents](classes/BranchContents.md)
- [Branches](classes/Branches.md)
- [Catalog](classes/Catalog.md)
- [Connection](classes/Connection.md)
- [HeaderProvider](classes/HeaderProvider.md)
- [Index](classes/Index.md)
@@ -34,6 +37,7 @@
- [MultiMatchQuery](classes/MultiMatchQuery.md)
- [NativeJsHeaderProvider](classes/NativeJsHeaderProvider.md)
- [OAuthHeaderProvider](classes/OAuthHeaderProvider.md)
- [OAuthSession](classes/OAuthSession.md)
- [PermutationBuilder](classes/PermutationBuilder.md)
- [PhraseQuery](classes/PhraseQuery.md)
- [Query](classes/Query.md)
@@ -61,6 +65,7 @@
- [BranchIndexSummary](interfaces/BranchIndexSummary.md)
- [BranchRowCountSummary](interfaces/BranchRowCountSummary.md)
- [BucketStats](interfaces/BucketStats.md)
- [CatalogOptions](interfaces/CatalogOptions.md)
- [CherryPickError](interfaces/CherryPickError.md)
- [CherryPickPreview](interfaces/CherryPickPreview.md)
- [CherryPickResult](interfaces/CherryPickResult.md)
@@ -87,6 +92,10 @@
- [FtsToken](interfaces/FtsToken.md)
- [FullTextQuery](interfaces/FullTextQuery.md)
- [FullTextSearchOptions](interfaces/FullTextSearchOptions.md)
- [FunctionErrorFragment](interfaces/FunctionErrorFragment.md)
- [FunctionErrorRecord](interfaces/FunctionErrorRecord.md)
- [FunctionErrors](interfaces/FunctionErrors.md)
- [FunctionErrorsOptions](interfaces/FunctionErrorsOptions.md)
- [GenerationStats](interfaces/GenerationStats.md)
- [HnswPqOptions](interfaces/HnswPqOptions.md)
- [HnswSqOptions](interfaces/HnswSqOptions.md)
@@ -96,9 +105,10 @@
- [IvfFlatOptions](interfaces/IvfFlatOptions.md)
- [IvfPqOptions](interfaces/IvfPqOptions.md)
- [IvfRqOptions](interfaces/IvfRqOptions.md)
- [JobDescription](interfaces/JobDescription.md)
- [JobEventsOptions](interfaces/JobEventsOptions.md)
- [JobFailureInfo](interfaces/JobFailureInfo.md)
- [JobInfo](interfaces/JobInfo.md)
- [ListDatabasesResponse](interfaces/ListDatabasesResponse.md)
- [ListNamespacesOptions](interfaces/ListNamespacesOptions.md)
- [ListNamespacesResponse](interfaces/ListNamespacesResponse.md)
- [ListTablesOptions](interfaces/ListTablesOptions.md)
@@ -121,6 +131,8 @@
- [RestNamespaceConfig](interfaces/RestNamespaceConfig.md)
- [RetryConfig](interfaces/RetryConfig.md)
- [ScannableOptions](interfaces/ScannableOptions.md)
- [SessionLogout](interfaces/SessionLogout.md)
- [SessionStatus](interfaces/SessionStatus.md)
- [ShuffleOptions](interfaces/ShuffleOptions.md)
- [SplitCalculatedOptions](interfaces/SplitCalculatedOptions.md)
- [SplitHashOptions](interfaces/SplitHashOptions.md)
@@ -130,12 +142,14 @@
- [TableStatistics](interfaces/TableStatistics.md)
- [TimeoutConfig](interfaces/TimeoutConfig.md)
- [TlsConfig](interfaces/TlsConfig.md)
- [TokenCacheOptions](interfaces/TokenCacheOptions.md)
- [TokenResponse](interfaces/TokenResponse.md)
- [TokenizeOptions](interfaces/TokenizeOptions.md)
- [UpdateFieldMetadataResult](interfaces/UpdateFieldMetadataResult.md)
- [UpdateOptions](interfaces/UpdateOptions.md)
- [UpdateResult](interfaces/UpdateResult.md)
- [Version](interfaces/Version.md)
- [ViewDescription](interfaces/ViewDescription.md)
- [WriteExecutionOptions](interfaces/WriteExecutionOptions.md)
- [WriteProgress](interfaces/WriteProgress.md)
@@ -143,8 +157,15 @@
- [AnalyzePlanDistributedMetrics](type-aliases/AnalyzePlanDistributedMetrics.md)
- [BaseTokenizer](type-aliases/BaseTokenizer.md)
- [BlobData](type-aliases/BlobData.md)
- [BlobInput](type-aliases/BlobInput.md)
- [BlobOptions](type-aliases/BlobOptions.md)
- [BlobRange](type-aliases/BlobRange.md)
- [BlobReadOptions](type-aliases/BlobReadOptions.md)
- [BlobUri](type-aliases/BlobUri.md)
- [Data](type-aliases/Data.md)
- [DataLike](type-aliases/DataLike.md)
- [DataTypeLike](type-aliases/DataTypeLike.md)
- [FieldLike](type-aliases/FieldLike.md)
- [IntoSql](type-aliases/IntoSql.md)
- [IntoVector](type-aliases/IntoVector.md)
@@ -158,10 +179,14 @@
## Functions
- [RecordBatchIterator](functions/RecordBatchIterator.md)
- [blob](functions/blob.md)
- [connect](functions/connect.md)
- [connectCatalog](functions/connectCatalog.md)
- [connectNamespace](functions/connectNamespace.md)
- [instrumentLanceDbMetrics](functions/instrumentLanceDbMetrics.md)
- [isBlobField](functions/isBlobField.md)
- [makeArrowTable](functions/makeArrowTable.md)
- [makeJsonField](functions/makeJsonField.md)
- [packBits](functions/packBits.md)
- [permutationBuilder](functions/permutationBuilder.md)
- [tokenize](functions/tokenize.md)
docs/src/js/interfaces/AddDataOptions.md
+16
View File
@@ -10,6 +10,22 @@ Options for adding data to a table.
## Properties
### allowExternalBlobOutsideBases?
```ts
optional allowExternalBlobOutsideBases: boolean;
```
Whether blob URIs outside registered external bases may be written.
Defaults to `false`. This option is supported only for local/native
tables; remote tables return an error when it is enabled. An enabled write
stores the absolute URI reference without registering a base or copying
the external object into the database. The object must remain accessible
when the blob is read later.
***
### mode
```ts
docs/src/js/interfaces/CatalogOptions.md
+81
View File
@@ -0,0 +1,81 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / CatalogOptions
# Interface: CatalogOptions
Options shared by a catalog and the database connections it returns.
## Extends
- `Omit`&lt;`NativeCatalogOptions`, `"oauthConfig"`&gt;
## Properties
### apiKey?
```ts
optional apiKey: string;
```
#### Inherited from
`Omit.apiKey`
***
### clientConfig?
```ts
optional clientConfig: ClientConfig;
```
#### Inherited from
`Omit.clientConfig`
***
### headerProvider?
```ts
optional headerProvider: HeaderProvider | () => Record<string, string> | Promise<Record<string, string>>;
```
Called for each request to supply authentication headers.
***
### oauthConfig?
```ts
optional oauthConfig: OAuthConfig;
```
***
### readConsistencyInterval?
```ts
optional readConsistencyInterval: number;
```
#### Inherited from
`Omit.readConsistencyInterval`
***
### sqlHostOverride?
```ts
optional sqlHostOverride: string;
```
SQL service endpoint inherited by database connections.
#### Inherited from
`Omit.sqlHostOverride`
docs/src/js/interfaces/CherryPickResult.md
+3 -3
View File
@@ -38,9 +38,9 @@ preview: CherryPickPreview;
```ts
status:
| "failed"
| "unknown"
| "ready"
| "failed"
| "notImplemented"
| "cherryPicked";
| "cherryPicked"
| "unknown";
```
docs/src/js/interfaces/ClientConfig.md
+4
View File
@@ -22,6 +22,10 @@ optional extraHeaders: Record<string, string>;
optional idDelimiter: string;
```
The delimiter joining a namespace path and a name into one object
identifier. `"$"` is the only supported value, and leaving this unset is
how to get it; anything else is rejected when the connection is created.
***
### retryConfig?
docs/src/js/interfaces/DropNamespaceOptions.md
+1 -1
View File
@@ -21,7 +21,7 @@ Refuse to drop if non-empty (restrict) or drop recursively (cascade).
### mode?
```ts
optional mode: "fail" | "skip";
optional mode: "skip" | "fail";
```
Whether to skip if the namespace doesn't exist, or fail.
docs/src/js/interfaces/FunctionErrorFragment.md
+42
View File
@@ -0,0 +1,42 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / FunctionErrorFragment
# Interface: FunctionErrorFragment
A fragment whose per-row error detail was capped: `rowsSkipped` rows
failed, of which only `rowsRecorded` have a record of their own.
## Properties
### fragmentId
```ts
fragmentId: number;
```
***
### jobId
```ts
jobId: string;
```
***
### rowsRecorded
```ts
rowsRecorded: number;
```
***
### rowsSkipped
```ts
rowsSkipped: number;
```
docs/src/js/interfaces/FunctionErrorRecord.md
+92
View File
@@ -0,0 +1,92 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / FunctionErrorRecord
# Interface: FunctionErrorRecord
One row a Function refresh skipped, as the server recorded it.
## Properties
### column
```ts
column: string;
```
***
### createdAtMillis
```ts
createdAtMillis: number;
```
***
### errorMessage
```ts
errorMessage: string;
```
***
### errorType
```ts
errorType: string;
```
***
### fragmentId
```ts
fragmentId: number;
```
***
### function
```ts
function: string;
```
***
### functionVersion
```ts
functionVersion: string;
```
***
### jobId
```ts
jobId: string;
```
***
### rowOffset?
```ts
optional rowOffset: number;
```
The row's offset within the fragment; absent when the fragment's
detail was capped.
***
### tableVersion
```ts
tableVersion: number;
```
docs/src/js/interfaces/FunctionErrors.md
+39
View File
@@ -0,0 +1,39 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / FunctionErrors
# Interface: FunctionErrors
A table's per-row Function errors.
## Properties
### fragments
```ts
fragments: FunctionErrorFragment[];
```
Fragments whose detail was capped.
***
### records
```ts
records: FunctionErrorRecord[];
```
The recorded rows, newest job first.
***
### truncated
```ts
truncated: boolean;
```
Whether the listing stopped at its limit.
docs/src/js/interfaces/FunctionErrorsOptions.md
+39
View File
@@ -0,0 +1,39 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / FunctionErrorsOptions
# Interface: FunctionErrorsOptions
Which per-row Function errors to list; every filter is optional.
## Properties
### column?
```ts
optional column: string;
```
Only errors on this column.
***
### jobId?
```ts
optional jobId: string;
```
Only errors recorded by this job.
***
### limit?
```ts
optional limit: number;
```
At most this many records (server default 10000, cap 100000).
docs/src/js/interfaces/HnswPqOptions.md
+1 -1
View File
@@ -118,7 +118,7 @@ Number of sub-vectors of PQ.
This value controls how much the vector is compressed during the quantization step.
The more sub vectors there are the less the vector is compressed. The default is
the dimension of the vector divided by 16. If the dimension is not evenly divisible
by 16 we use the dimension divded by 8.
by 16 we use the dimension divided by 8.
The above two cases are highly preferred. Having 8 or 16 values per subvector allows
us to use efficient SIMD instructions.
docs/src/js/interfaces/IndexOptions.md
+1 -1
View File
@@ -16,7 +16,7 @@ optional config: Index;
Advanced index configuration
This option allows you to specify a specfic index to create and also
This option allows you to specify a specific index to create and also
allows you to pass in configuration for training the index.
See the static methods on Index for details on the various index types.
docs/src/js/interfaces/IvfPqOptions.md
+1 -1
View File
@@ -112,7 +112,7 @@ Number of sub-vectors of PQ.
This value controls how much the vector is compressed during the quantization step.
The more sub vectors there are the less the vector is compressed. The default is
the dimension of the vector divided by 16. If the dimension is not evenly divisible
by 16 we use the dimension divded by 8.
by 16 we use the dimension divided by 8.
The above two cases are highly preferred. Having 8 or 16 values per subvector allows
us to use efficient SIMD instructions.
docs/src/js/interfaces/JobDescription.md
-66
View File
@@ -1,66 +0,0 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / JobDescription
# Interface: JobDescription
A described job from `Connection.getJob`.
## Properties
### creationMs
```ts
creationMs: number;
```
When the job was created, in milliseconds since the epoch.
***
### failure?
```ts
optional failure: JobFailureInfo;
```
Why the job failed, when the job is failed and the server reports a
reason.
***
### jobId
```ts
jobId: string;
```
***
### jobType
```ts
jobType: string;
```
***
### specJson?
```ts
optional specJson: string;
```
The job-type-specific specification as a JSON string, when present.
***
### state
```ts
state: string;
```
Lifecycle state: "running", "finished", "failed", or "cancelled".
docs/src/js/interfaces/JobEventsOptions.md
+29
View File
@@ -0,0 +1,29 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / JobEventsOptions
# Interface: JobEventsOptions
Which of a job's events [Job.events](../classes/Job.md#events) returns.
## Properties
### filter?
```ts
optional filter: string;
```
SQL-like filter over the event columns.
***
### limit?
```ts
optional limit: number;
```
Maximum event rows to return, up to the server maximum of 10,000.
docs/src/js/interfaces/JobInfo.md
+1 -1
View File
@@ -26,7 +26,7 @@ When the job was created, in milliseconds since the epoch.
jobId: string;
```
The job id -- what `Connection.getJob` and `Connection.cancelJob`
The job id -- what `Connection.openJob` and `Connection.cancelJob`
accept.
***
docs/src/js/interfaces/ListDatabasesResponse.md
+23
View File
@@ -0,0 +1,23 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / ListDatabasesResponse
# Interface: ListDatabasesResponse
## Properties
### databases
```ts
databases: string[];
```
***
### pageToken?
```ts
optional pageToken: string;
```
docs/src/js/interfaces/MaterializedViewDefinition.md
+7 -54
View File
@@ -6,64 +6,17 @@
# Interface: MaterializedViewDefinition
The query that defines a materialized view.
The query that defines a materialized view, as stored:
`SELECT columns FROM [ns.]table [, function(args) AS alias | , UNNEST(column) AS alias]
[WHERE predicate] [GROUP BY expr, ...] [LIMIT n]`. A Function in `FROM`
position yields one row per element it returns.
## Properties
### filter?
### query
```ts
optional filter: string;
query: string;
```
SQL predicate selecting the source rows the view holds.
***
### inputs
```ts
inputs: string[];
```
Source columns the projections and filter read.
***
### limit?
```ts
optional limit: number;
```
Cap on the number of rows the view holds.
***
### projections
```ts
projections: [string, string][];
```
`[output column, SQL expression]` pairs, in view schema order.
***
### sourceNamespace
```ts
sourceNamespace: string[];
```
Namespace holding the source table; empty is the root namespace.
***
### sourceTable
```ts
sourceTable: string;
```
Name of the source table, in the same database as the view.
The defining query, in the canonical spelling the server stores.
docs/src/js/interfaces/NativeOAuthConfig.md
+76 -1
View File
@@ -15,6 +15,39 @@ All token acquisition and refresh is handled in the Rust layer.
## Properties
### audience?
```ts
optional audience: string;
```
Optional provider-specific audience for authorization and token requests.
***
### callbackPort?
```ts
optional callbackPort: number;
```
Port for the authorization_code loopback callback server.
***
### clientAuthMethod?
```ts
optional clientAuthMethod: string;
```
How the client authenticates to the token endpoint: "none",
"client_secret_basic", or "client_secret_post". Defaults to
"client_secret_basic" when a client secret is set, and "none" for
public clients.
***
### clientId
```ts
@@ -41,7 +74,8 @@ Client secret (required for client_credentials).
optional flow: string;
```
Authentication flow: "client_credentials" or "azure_managed_identity"
Authentication flow: "client_credentials", "authorization_code",
"device_code", or "azure_managed_identity"
***
@@ -66,6 +100,16 @@ Client ID for user-assigned managed identity (azure_managed_identity).
***
### redirectUri?
```ts
optional redirectUri: string;
```
Loopback redirect URI for authorization_code.
***
### refreshBufferSecs?
```ts
@@ -78,6 +122,16 @@ the TTL, each request refreshes the token.
***
### resource?
```ts
optional resource: string;
```
Optional resource indicator for authorization and token requests.
***
### scopes
```ts
@@ -86,3 +140,24 @@ scopes: string[];
OAuth scopes to request. For Azure managed identity, exactly one scope
or resource is required. For example: `["api://{app_id}/.default"]`
***
### tokenCache?
```ts
optional tokenCache: TokenCacheOptions;
```
Opt in to the persistent token cache so short-lived processes reuse
one session. Only refresh tokens are persisted.
***
### usePkce?
```ts
optional usePkce: boolean;
```
Whether authorization_code uses S256 PKCE (default: true).
docs/src/js/interfaces/OAuthConfig.md
+106
View File
@@ -26,6 +26,18 @@ const config: OAuthConfig = {
};
```
Providers requiring an explicit target can set `resource` and/or `audience`:
```typescript
const targeted: OAuthConfig = {
issuerUrl: "https://issuer.example.com",
clientId: "app-id",
clientSecret: "secret",
scopes: ["read"],
resource: "https://api.example.com",
audience: "lancedb-api",
};
```
```typescript
const config: OAuthConfig = {
issuerUrl: "https://login.microsoftonline.com/{tenant}/v2.0",
@@ -35,8 +47,57 @@ const config: OAuthConfig = {
};
```
The authorization URL is written to stderr before LanceDB tries to open a
browser, so it can be copied in headless environments.
```typescript
const config: OAuthConfig = {
issuerUrl: "https://login.microsoftonline.com/{tenant}/v2.0",
clientId: "app-id",
scopes: ["openid", "api://lancedb-api/access"],
flow: OAuthFlowType.AuthorizationCode,
};
```
Device Authorization writes the verification URL and user code to stderr
before polling begins.
## Properties
### audience?
```ts
optional audience: string;
```
Provider-specific audience, forwarded to authorization and token endpoints,
including refresh requests. Not supported for Azure managed identity.
***
### callbackPort?
```ts
optional callbackPort: number;
```
Port for the AuthorizationCode loopback callback server (default: 8400).
***
### clientAuthMethod?
```ts
optional clientAuthMethod: ClientAuthMethod;
```
How the client authenticates to the token endpoint (default: auto).
With a `clientSecret` the default is `ClientAuthMethod.ClientSecretBasic`,
which matches the RFC 6749 recommendation and the default configuration
of Okta confidential applications; without a secret the client is public
and no client authentication is sent.
***
### clientId
```ts
@@ -88,6 +149,16 @@ Client ID for user-assigned managed identity (AzureManagedIdentity).
***
### redirectUri?
```ts
optional redirectUri: string;
```
Loopback redirect URI for AuthorizationCode.
***
### refreshBufferSecs?
```ts
@@ -100,6 +171,18 @@ the TTL, each request refreshes the token.
***
### resource?
```ts
optional resource: string;
```
Resource indicator (RFC 8707), forwarded verbatim to authorization and token
endpoints, including refresh requests. Must be an absolute URI without a
fragment. Not supported for Azure managed identity.
***
### scopes
```ts
@@ -109,3 +192,26 @@ scopes: string[];
OAuth scopes to request.
For Azure managed identity, exactly one scope or resource is required.
For example: `["api://{app_id}/.default"]`
***
### tokenCache?
```ts
optional tokenCache: TokenCacheOptions;
```
Opt in to the persistent token cache so short-lived processes reuse one
session. Only refresh tokens are persisted. Only supported by
AuthorizationCode and DeviceCode; Azure managed identity is rejected.
Default: unset (memory only).
***
### usePkce?
```ts
optional usePkce: boolean;
```
Protect AuthorizationCode with S256 PKCE (default: true).
docs/src/js/interfaces/OptimizeOptions.md
+2 -1
View File
@@ -26,7 +26,8 @@ const olderThan = new Date();
olderThan.setDate(olderThan.getDate() - 1));
tbl.optimize({cleanupOlderThan: olderThan});
// Delete all versions except the current version
// Delete versions committed before this point. Versions created by the
// optimize call itself are newer than the cutoff and will be retained.
tbl.optimize({cleanupOlderThan: new Date()});
```
docs/src/js/interfaces/SessionLogout.md
+20
View File
@@ -0,0 +1,20 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / SessionLogout
# Interface: SessionLogout
Result of [OAuthSession.logout](../classes/OAuthSession.md#logout).
## Properties
### removed
```ts
removed: boolean;
```
Whether a cached credential was removed. `false` means no matching
session was cached; logout is idempotent.
docs/src/js/interfaces/SessionStatus.md
+94
View File
@@ -0,0 +1,94 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / SessionStatus
# Interface: SessionStatus
Safe, non-secret view of a cached OAuth session, returned by
[OAuthSession.status](../classes/OAuthSession.md#status) and [OAuthSession.login](../classes/OAuthSession.md#login).
## Properties
### audience?
```ts
optional audience: string;
```
Provider-specific audience used to obtain the cached session, if configured.
***
### clientId
```ts
clientId: string;
```
Client ID of the cached session.
***
### flow
```ts
flow: string;
```
Flow that produced the cached session.
***
### issuerUrl
```ts
issuerUrl: string;
```
Canonical issuer URL of the cached session.
***
### obtainedAt?
```ts
optional obtainedAt: number;
```
When the cached session was obtained, as Unix seconds.
***
### refreshable
```ts
refreshable: boolean;
```
Whether a cached session exists that can obtain tokens without
interactive authentication. Because access tokens are not persisted,
this is `true` exactly when a refresh token is cached; the next
connection refreshes with it rather than opening a browser or device
prompt.
***
### resource?
```ts
optional resource: string;
```
Resource indicator used to obtain the cached session, if configured.
***
### scopes
```ts
scopes: string[];
```
Canonical (sorted, de-duplicated) scope set of the cached session.
docs/src/js/interfaces/TokenCacheOptions.md
+41
View File
@@ -0,0 +1,41 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / TokenCacheOptions
# Interface: TokenCacheOptions
Options for the persistent OAuth token cache.
The cache is opt-in: it is only used when set as `tokenCache` on
[OAuthConfig](OAuthConfig.md). Only refresh tokens are persisted, in a private
directory with owner-only permissions, so short-lived processes can reuse
an authenticated session instead of re-prompting on every start.
Multiple identities (issuer, client, scopes, resource, audience, flow, client authentication)
get separate cache entries. Within one identity the most recent login wins.
## Properties
### cacheDir?
```ts
optional cacheDir: string;
```
Directory that holds cached credentials. Defaults to
`$XDG_CACHE_HOME/lancedb/oauth`, `$HOME/.cache/lancedb/oauth` on Unix,
or `%LOCALAPPDATA%\\lancedb\\oauth` on Windows. The directory is created
with owner-only permissions (`0700`) when missing.
***
### lockTimeoutSecs?
```ts
optional lockTimeoutSecs: number;
```
How long to wait for the cross-process refresh lock before failing, in
seconds (default: 30).
docs/src/js/interfaces/ViewDescription.md
+78
View File
@@ -0,0 +1,78 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / ViewDescription
# Interface: ViewDescription
What a database records about one view.
A view holds no rows: it stores the statement that defines it and the
schema that statement resolved to, so a reader sees its sources as they
are at read time.
## Properties
### defaultDatabase
```ts
defaultDatabase: string;
```
The database that unqualified table names in [query](ViewDescription.md#query) resolve against.
***
### defaultNamespacePath
```ts
defaultNamespacePath: string[];
```
The namespace path those unqualified names resolve against; empty is the
root namespace.
Recorded with the view because it outlives the session that declared it:
a reader resolving the query against its own default namespace could read
a different table than the view was defined over.
***
### name
```ts
name: string;
```
The view's name within its namespace.
***
### namespacePath
```ts
namespacePath: string[];
```
The namespace holding the view; empty is the root namespace.
***
### query
```ts
query: string;
```
The defining query, as the database stores it.
***
### schema
```ts
schema: Schema<any>;
```
The schema the defining query resolved to when the view was created.
docs/src/js/namespaces/embedding/classes/EmbeddingFunction.md
+2 -2
View File
@@ -52,7 +52,7 @@ new EmbeddingFunction<T, M>(): EmbeddingFunction<T, M>
### computeQueryEmbeddings()
```ts
computeQueryEmbeddings(data): Promise<number[] | Uint8Array | Float32Array | Float64Array>
computeQueryEmbeddings(data): Promise<Float32Array | Float64Array | Uint8Array | number[]>
```
Compute the embeddings for a single query
@@ -63,7 +63,7 @@ Compute the embeddings for a single query
#### Returns
`Promise`&lt;`number`[] \| `Uint8Array` \| `Float32Array` \| `Float64Array`&gt;
`Promise`&lt;`Float32Array` \| `Float64Array` \| `Uint8Array` \| `number`[]&gt;
***
docs/src/js/namespaces/embedding/classes/TextEmbeddingFunction.md
+2 -2
View File
@@ -37,7 +37,7 @@ new TextEmbeddingFunction<M>(): TextEmbeddingFunction<M>
### computeQueryEmbeddings()
```ts
computeQueryEmbeddings(data): Promise<number[] | Uint8Array | Float32Array | Float64Array>
computeQueryEmbeddings(data): Promise<Float32Array | Float64Array | Uint8Array | number[]>
```
Compute the embeddings for a single query
@@ -48,7 +48,7 @@ Compute the embeddings for a single query
#### Returns
`Promise`&lt;`number`[] \| `Uint8Array` \| `Float32Array` \| `Float64Array`&gt;
`Promise`&lt;`Float32Array` \| `Float64Array` \| `Uint8Array` \| `number`[]&gt;
#### Overrides
docs/src/js/type-aliases/BlobData.md
+16
View File
@@ -0,0 +1,16 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobData
# Type Alias: BlobData
```ts
type BlobData: Buffer | Uint8Array | ArrayBuffer | Blob;
```
Bytes accepted for a blob value. `ArrayBuffer` is wrapped without copying.
`Blob` and `File` are read with `arrayBuffer()`, which only the async write
paths ([Connection.createTable](../classes/Connection.md#createtable), [Table.add](../classes/Table.md#add),
[Table.mergeInsert](../classes/Table.md#mergeinsert)) can do.
docs/src/js/type-aliases/BlobInput.md
+36
View File
@@ -0,0 +1,36 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobInput
# Type Alias: BlobInput
```ts
type BlobInput:
| BlobData
| BlobUri
| object
| object
| null
| undefined;
```
A value accepted for a `lance.blob.v2` column (see [blob](../functions/blob.md)).
Either inline bytes, a URI pointing at external bytes, or a struct that sets
exactly one of `data` or `uri`. `null` and `undefined` write a null blob.
## Example
```ts
import { pathToFileURL } from "node:url";
import type { BlobInput } from "@lancedb/lancedb";
const rows: { id: bigint; image: BlobInput }[] = [
{ id: 1n, image: await (await fetch(url)).arrayBuffer() },
{ id: 2n, image: pathToFileURL("/data/cat.png") },
{ id: 3n, image: { data: new Blob(["hello"]) } },
];
await table.add(rows);
```
docs/src/js/type-aliases/BlobOptions.md
+48
View File
@@ -0,0 +1,48 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobOptions
# Type Alias: BlobOptions
```ts
type BlobOptions: object;
```
## Type declaration
### dedicatedSizeThreshold?
```ts
optional dedicatedSizeThreshold: number;
```
Max payload bytes stored in a packed sidecar before a dedicated file. Must
be a positive safe integer.
### inlineSizeThreshold?
```ts
optional inlineSizeThreshold: number;
```
Max payload bytes kept inline in the data file. Zero is allowed. Must be a
safe integer.
### nullable?
```ts
optional nullable: boolean;
```
Defaults to true.
### packFileSizeThreshold?
```ts
optional packFileSizeThreshold: number;
```
Max bytes in one packed sidecar before starting another. Must be a positive
safe integer.
docs/src/js/type-aliases/BlobRange.md
+27
View File
@@ -0,0 +1,27 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobRange
# Type Alias: BlobRange
```ts
type BlobRange: object;
```
A half-open byte range `[start, end)` for [BlobFile.readRanges](../classes/BlobFile.md#readranges).
## Type declaration
### end
```ts
end: bigint;
```
### start
```ts
start: bigint;
```
docs/src/js/type-aliases/BlobReadOptions.md
+24
View File
@@ -0,0 +1,24 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobReadOptions
# Type Alias: BlobReadOptions
```ts
type BlobReadOptions: object;
```
Options for blob reads.
## Type declaration
### signal?
```ts
optional signal: AbortSignal;
```
Cancels the read. The call rejects with `signal.reason`, and the native
read stops, including in-flight requests to a remote table.
docs/src/js/type-aliases/BlobUri.md
+13
View File
@@ -0,0 +1,13 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobUri
# Type Alias: BlobUri
```ts
type BlobUri: string | URL;
```
A URI accepted for a blob value. A `URL` is stored as its `href`.
docs/src/js/type-aliases/DataTypeLike.md
+18
View File
@@ -0,0 +1,18 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / DataTypeLike
# Type Alias: DataTypeLike
```ts
type DataTypeLike: DataType | object;
```
A `DataType` from any copy or version of apache-arrow.
Arrow 21 brands its classes with `unique symbol` properties, so a type
object from a second copy of the library no longer satisfies the `DataType`
type of this one even though it is structurally identical. Inputs that only
need to be sanitized accept this looser shape instead.
docs/src/python/python.md
+154
View File
@@ -28,6 +28,70 @@ is also an [asynchronous API client](#connections-asynchronous).
::: lancedb.Session
## Catalogs (Synchronous)
Remote catalogs manage databases through a server's root namespace. Opened databases
are ordinary connections. Dropping a database requires it to be empty.
::: lancedb.connect_catalog
::: lancedb.catalog.Catalog
::: lancedb.catalog.ListDatabasesResponse
## Remote SQL
Submit SQL against a remote LanceDB database through the connection.
The connected database and `default_namespace_path=["public"]` are used for
unqualified tables. Fully qualified references can still query other databases
and namespaces available to the same deployment. `execute_query` returns a
reader as soon as its initial result stream is available. `execute_query_async`
returns a query handle immediately; use it to inspect progress, open a reader,
or cancel the query. The SQL client is initialized by the first query and
retained for the lifetime of the remote connection. Query ids are random,
connection-scoped references rather than encoded SQL or durable resume tokens:
```python
import lancedb
db = lancedb.connect(
"db://analytics",
api_key="ldb_...",
host_override="https://api.example.com",
sql_host_override="grpc+tls://sql.example.com:10026",
)
reader = db.execute_query(
"""
SELECT events.id, accounts.name
FROM analytics.public.events AS events
JOIN users.public.accounts AS accounts ON events.user_id = accounts.id
""",
default_namespace_path=["public"],
)
for batch in reader:
print(batch.num_rows)
query = db.execute_query_async("SELECT * FROM events")
print(query.id)
print(query.describe().status)
for batch in query.reader():
print(batch.num_rows)
# The async connection exposes the same lifecycle without blocking:
# async_db = await lancedb.connect_async(
# "db://analytics",
# api_key="ldb_...",
# host_override="https://api.example.com",
# sql_host_override="grpc+tls://sql.example.com:10026",
# )
# reader = await async_db.execute_query("SELECT * FROM events")
# query = await async_db.execute_query_async("SELECT * FROM events")
# description = await async_db.describe_query(query.id)
# async for batch in await query.reader():
# print(batch.num_rows)
# await query.cancel()
```
## Namespaces (Synchronous)
A namespace-backed connection resolves tables through a
@@ -72,6 +136,10 @@ listing a storage directory.
::: lancedb.functions.UdfDefinition
::: lancedb.secrets.EnvVarSecret
::: lancedb.secrets.SecretInfo
::: lancedb.functions.FunctionRegistrationRequest
::: lancedb.functions.FunctionArtifactRequest
@@ -80,6 +148,8 @@ listing a storage directory.
::: lancedb.functions.PythonAdapterSpec
::: lancedb.functions.FunctionImage
::: lancedb.functions.FunctionVersion
::: lancedb.functions.PythonRuntimeSpec
@@ -94,20 +164,71 @@ listing a storage directory.
::: lancedb.functions.OutputMapping
::: lancedb.functions.AssignmentMapping
::: lancedb.functions.FunctionBinding
::: lancedb.functions.RefreshColumnResult
::: lancedb.FunctionErrors
::: lancedb.FunctionErrorRecord
::: lancedb.FunctionErrorFragment
::: lancedb.job.Job
::: lancedb.job.AsyncJob
::: lancedb.job.JobInfo
::: lancedb.job.JobDescription
::: lancedb.job.JobFailureInfo
::: lancedb.sql.Query
::: lancedb.sql.AsyncQuery
::: lancedb.sql.QueryDescription
## Materialized Views (Synchronous)
::: lancedb.materialized_view.MaterializedView
::: lancedb.materialized_view.MaterializedViewDefinition
::: lancedb.vector_dedup
::: lancedb.materialized_view.VectorDedupSource
For an existing indexed source, declare and materialize a dedup result without
writing SQL. Creation captures the source version once and waits for the complete
result; remote connections use the service's MV jobs.
```python
from lancedb import vector_dedup
view = db.create_materialized_view(
"images_clean",
vector_dedup("images", column="phash", distance_threshold=4),
)
cleaned = view.table
```
To submit without waiting, call `db.create_materialized_view_async(...)` with the
same source expression and wait on the returned job. To declare only, pass
`with_no_data=True`, then use `view.refresh_async().wait()` when ready. Pass
`dataset_version=...` to `vector_dedup` for an explicit snapshot. The default
policy retains direct representatives (A-B-C without A-C keeps A and C), preserves
the source, and materializes its original columns. Only within-segment/partition
pairs are covered, using the index's distance representation.
## Views
::: lancedb.view.ViewDescription
## Expressions
Type-safe expression builder for filters and projections. Use these instead
@@ -249,6 +370,12 @@ still work. Queries return descriptors. Call
::: lancedb.exceptions.MissingColumnError
::: lancedb.exceptions.JobNotFoundError
::: lancedb.exceptions.JobFailedError
::: lancedb.exceptions.JobCancelledError
## Integrations
## Pydantic
@@ -279,6 +406,29 @@ still work. Queries return descriptors. Call
## Reranking
`TypeSafeReranker` supports opt-in request batching:
```python
from lancedb.rerankers import TypeSafeReranker
reranker = TypeSafeReranker(batch_size=40, max_concurrency=8)
```
The default `batch_size=1` keeps the query and document in request state and
sends one request per non-null candidate. With `batch_size=40`, 80 non-null
candidates require two requests. `max_concurrency` still limits simultaneous
requests, and the SDK handles retries.
In batched mode, state contains only `{"query": query}`. Each independent
question contains `{"question": instructions, "document": document}` in its
structured instructions, so it sees only its own document and the shared query.
Custom instructions and criteria are kept verbatim: adapt prompts that explicitly
reference request-state fields such as `state.document` before enabling batching.
Null documents retain a zero score without an API call; empty strings are scored.
API failures, mismatched answer IDs, and invalid probabilities raise errors.
Batching changes the payload and can affect model scores; compare quality and
latency on your workload before opting in.
::: lancedb.rerankers
options:
show_root_heading: false
@@ -286,6 +436,10 @@ still work. Queries return descriptors. Call
## Connections (Asynchronous)
::: lancedb.connect_catalog_async
::: lancedb.catalog.AsyncCatalog
Connections represent a connection to a LanceDb database and
can be used to create, list, or open tables.
java/lancedb-core/pom.xml
+1 -1
View File
@@ -8,7 +8,7 @@
<parent>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.39.0-beta.1</version>
<version>0.40.0-beta.12</version>
<relativePath>../pom.xml</relativePath>
</parent>
java/pom.xml
+2 -2
View File
@@ -6,7 +6,7 @@
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.39.0-beta.1</version>
<version>0.40.0-beta.12</version>
<packaging>pom</packaging>
<name>${project.artifactId}</name>
<description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<arrow.version>15.0.0</arrow.version>
<lance-core.version>12.0.0-beta.11</lance-core.version>
<lance-core.version>13.1.0-beta.1</lance-core.version>
<spotless.skip>false</spotless.skip>
<spotless.version>2.30.0</spotless.version>
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
nodejs/CLAUDE.md
-1
View File
@@ -1 +0,0 @@
AGENTS.md
nodejs/Cargo.toml
+4 -3
View File
@@ -1,7 +1,7 @@
[package]
name = "lancedb-nodejs"
edition.workspace = true
version = "0.39.0-beta.1"
version = "0.40.0-beta.12"
publish = false
license.workspace = true
description.workspace = true
@@ -37,8 +37,9 @@ lzma-sys = { version = "0.1", features = ["static"] }
log.workspace = true
# Pin to resolve build failures; update periodically for security patches.
aws-lc-sys = "=0.40.0"
aws-lc-rs = "=1.16.3"
# rustls >= 0.23.45 (RUSTSEC-2026-0285) needs aws-lc-rs >= 1.18.
aws-lc-sys = "=0.45.0"
aws-lc-rs = "=1.18.1"
[build-dependencies]
napi-build = "2.3.1"
nodejs/NODEJS_THIRD_PARTY_LICENSES.md
+8
View File
@@ -275,6 +275,7 @@
[@types/node@20.16.10](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/node@20.17.9](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/node@22.7.4](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/node@25.9.8](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/semver@7.5.6](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/stack-utils@2.0.3](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/tmp@0.2.6](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
@@ -306,6 +307,9 @@
[apache-arrow@16.0.0](https://github.com/apache/arrow) - Apache-2.0
[apache-arrow@17.0.0](https://github.com/apache/arrow) - Apache-2.0
[apache-arrow@18.0.0](https://github.com/apache/arrow) - Apache-2.0
[apache-arrow@19.0.1](https://github.com/apache/arrow) - Apache-2.0
[apache-arrow@20.0.0](https://github.com/apache/arrow) - Apache-2.0
[apache-arrow@21.2.0](https://github.com/apache/arrow-js) - Apache-2.0
[argparse@1.0.10](https://github.com/nodeca/argparse) - MIT
[argparse@2.0.1](https://github.com/nodeca/argparse) - Python-2.0
[array-back@3.1.0](https://github.com/75lb/array-back) - MIT
@@ -348,6 +352,7 @@
[color@4.2.3](https://github.com/Qix-/color) - MIT
[combined-stream@1.0.8](https://github.com/felixge/node-combined-stream) - MIT
[command-line-args@5.2.1](https://github.com/75lb/command-line-args) - MIT
[command-line-args@6.0.2](https://github.com/75lb/command-line-args) - MIT
[command-line-usage@7.0.1](https://github.com/75lb/command-line-usage) - MIT
[concat-map@0.0.1](https://github.com/substack/node-concat-map) - MIT
[convert-source-map@2.0.0](https://github.com/thlorenz/convert-source-map) - MIT
@@ -398,12 +403,14 @@
[file-entry-cache@6.0.1](https://github.com/royriojas/file-entry-cache) - MIT
[fill-range@7.1.1](https://github.com/jonschlinkert/fill-range) - MIT
[find-replace@3.0.0](https://github.com/75lb/find-replace) - MIT
[find-replace@5.0.2](https://github.com/75lb/find-replace) - MIT
[find-up@4.1.0](https://github.com/sindresorhus/find-up) - MIT
[find-up@5.0.0](https://github.com/sindresorhus/find-up) - MIT
[flat-cache@3.2.0](https://github.com/jaredwray/flat-cache) - MIT
[flatbuffers@1.12.0](https://github.com/google/flatbuffers) - Apache*
[flatbuffers@23.5.26](https://github.com/google/flatbuffers) - Apache*
[flatbuffers@24.3.25](https://github.com/google/flatbuffers) - Apache-2.0
[flatbuffers@25.9.23](https://github.com/google/flatbuffers) - Apache-2.0
[flatted@3.2.9](https://github.com/WebReflection/flatted) - ISC
[follow-redirects@1.15.6](https://github.com/follow-redirects/follow-redirects) - MIT
[foreground-child@3.3.0](https://github.com/tapjs/foreground-child) - ISC
@@ -644,6 +651,7 @@
[uc.micro@2.1.0](https://github.com/markdown-it/uc.micro) - MIT
[undici-types@5.26.5](https://github.com/nodejs/undici) - MIT
[undici-types@6.19.8](https://github.com/nodejs/undici) - MIT
[undici-types@7.24.6](https://github.com/nodejs/undici) - MIT
[update-browserslist-db@1.0.13](https://github.com/browserslist/update-db) - MIT
[uri-js@4.4.1](https://github.com/garycourt/uri-js) - BSD-2-Clause
[uuid@9.0.1](https://github.com/uuidjs/uuid) - MIT
nodejs/__test__/arrow.test.ts
+1487 -1378
View File
File diff suppressed because it is too large Load Diff
nodejs/__test__/arrow_versions.ts
+39
View File
@@ -0,0 +1,39 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// Every Apache Arrow release the package accepts as a peer dependency, as
// separately installed copies so the tests can feed objects built by one
// Arrow version into a LanceDB running on another. When a new Arrow major
// is released, add the alias to `package.json` and extend this list; the
// test files pick the change up from here.
import * as arrow15 from "apache-arrow-15";
import * as arrow16 from "apache-arrow-16";
import * as arrow17 from "apache-arrow-17";
import * as arrow18 from "apache-arrow-18";
import * as arrow19 from "apache-arrow-19";
import * as arrow20 from "apache-arrow-20";
import * as arrow21 from "apache-arrow-21";
export { arrow15, arrow16, arrow17, arrow18, arrow19, arrow20, arrow21 };
export type ApacheArrow =
| typeof arrow15
| typeof arrow16
| typeof arrow17
| typeof arrow18
| typeof arrow19
| typeof arrow20
| typeof arrow21;
export const arrowVersions: ApacheArrow[] = [
arrow15,
arrow16,
arrow17,
arrow18,
arrow19,
arrow20,
arrow21,
];
/** The newest supported Arrow release, for tests that only need one. */
export const latestArrow = arrow21;
nodejs/__test__/blob.test.ts
+509
View File
@@ -0,0 +1,509 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import {
Field,
Float,
Float32,
Int32,
Int64,
List,
Schema,
Struct,
Utf8,
} from "apache-arrow";
import {
convertToTable,
makeArrowTable,
resolveBlobInputs,
} from "../lancedb/arrow";
import { BlobFile, blob, coerceBlobValue, isBlobField } from "../lancedb/blob";
import { EmbeddingFunction, LanceSchema } from "../lancedb/embedding";
import { getRegistry } from "../lancedb/embedding/registry";
describe("blob()", () => {
it("marks the field as lance.blob.v2", () => {
const field = blob("image", { nullable: false });
expect(field.nullable).toBe(false);
expect(isBlobField(field)).toBe(true);
expect(field.metadata.get("ARROW:extension:name")).toBe("lance.blob.v2");
});
it("writes encoding thresholds as field metadata", () => {
const field = blob("video", {
inlineSizeThreshold: 1024,
dedicatedSizeThreshold: 2 * 1024 * 1024,
packFileSizeThreshold: 64 * 1024 * 1024,
});
expect(
field.metadata.get("lance-encoding:blob-inline-size-threshold"),
).toBe("1024");
expect(
field.metadata.get("lance-encoding:blob-dedicated-size-threshold"),
).toBe(String(2 * 1024 * 1024));
expect(
field.metadata.get("lance-encoding:blob-pack-file-size-threshold"),
).toBe(String(64 * 1024 * 1024));
});
it("rejects invalid thresholds", () => {
expect(() => blob("image", { inlineSizeThreshold: -1 })).toThrow(
/inlineSizeThreshold must be non-negative/,
);
expect(() => blob("image", { dedicatedSizeThreshold: 0 })).toThrow(
/dedicatedSizeThreshold must be positive/,
);
expect(() => blob("image", { packFileSizeThreshold: 1.5 })).toThrow(
/packFileSizeThreshold must be a safe integer/,
);
expect(() =>
blob("image", { dedicatedSizeThreshold: Number.MAX_SAFE_INTEGER + 1 }),
).toThrow(/dedicatedSizeThreshold must be a safe integer/);
});
});
class BlobStruct {
constructor(
public data?: unknown,
public uri?: unknown,
) {}
}
describe("coerceBlobValue", () => {
it.each([
["Buffer", Buffer.from("x"), { data: Buffer.from("x"), uri: null }],
[
"Uint8Array",
new Uint8Array([120]),
{ data: new Uint8Array([120]), uri: null },
],
["URI string", "s3://bucket/key", { data: null, uri: "s3://bucket/key" }],
[
"data struct",
{ data: Buffer.from("y") },
{ data: Buffer.from("y"), uri: null },
],
[
"uri struct",
{ uri: "s3://bucket/key" },
{ data: null, uri: "s3://bucket/key" },
],
[
"ArrayBuffer",
new Uint8Array([120]).buffer,
{ data: new Uint8Array([120]), uri: null },
],
[
"URL",
new URL("https://example.com/blob"),
{ data: null, uri: "https://example.com/blob" },
],
[
"data struct with ArrayBuffer",
{ data: new Uint8Array([121]).buffer },
{ data: new Uint8Array([121]), uri: null },
],
[
"uri struct with URL",
{ uri: new URL("file:///tmp/a.png") },
{ data: null, uri: "file:///tmp/a.png" },
],
[
"class instance with Uint8Array data",
new BlobStruct(new Uint8Array([122])),
{ data: new Uint8Array([122]), uri: null },
],
[
"class instance with ArrayBuffer data",
new BlobStruct(new Uint8Array([122]).buffer),
{ data: new Uint8Array([122]), uri: null },
],
[
"class instance with URL uri",
new BlobStruct(undefined, new URL("s3://bucket/key")),
{ data: null, uri: "s3://bucket/key" },
],
["null", null, null],
])("accepts %s", (_name, input, expected) => {
expect(coerceBlobValue(input)).toEqual(expected);
});
it.each([
["empty URI", "", /uri cannot be empty/],
["object without data or uri", { position: 0 }, /data' or 'uri/],
[
"Int16Array",
new Int16Array([1]),
/Blob data must be Buffer, Uint8Array, ArrayBuffer, or Blob/,
],
[
"both data and uri",
{ data: Buffer.from("y"), uri: "s3://bucket/key" },
/exactly one of 'data' or 'uri'/,
],
[
"neither data nor uri",
{ data: null, uri: null },
/exactly one of 'data' or 'uri'/,
],
["Blob", new Blob(["x"]), /must be read asynchronously/],
["data struct with Blob", { data: new Blob(["x"]) }, /asynchronously/],
[
"class instance with Blob data",
new BlobStruct(new Blob(["x"])),
/asynchronously/,
],
[
"unknown struct field",
{ data: Buffer.from("y"), mime: "image/png" },
/Blob struct values only support 'data' and 'uri', got 'mime'/,
],
[
"misspelled struct field",
{ uri: "s3://bucket/key", dat: Buffer.from("y") },
/got 'dat'/,
],
[
"SharedArrayBuffer",
new SharedArrayBuffer(2),
/Blob column values must be Buffer, Uint8Array, ArrayBuffer, Blob/,
],
["Date", new Date(0), /Blob column values must be/],
])("rejects %s", (_name, input, message) => {
expect(() => coerceBlobValue(input)).toThrow(message);
});
});
describe("resolveBlobInputs", () => {
it("reads Blob and File values in blob columns only", async () => {
const schema = new Schema([
new Field("name", new Utf8(), true),
blob("image"),
new Field("images", new List(blob("image")), true),
]);
const shared = new Blob(["same"]);
const notABlobColumn = new Blob(["other"]);
const data = [
{
name: notABlobColumn,
image: new File(["file"], "a.txt"),
images: [shared, { data: shared }, null],
},
{ name: "x", image: Buffer.from("buf"), images: null },
];
const resolved = await resolveBlobInputs(data, schema);
expect(resolved[0].name).toBe(notABlobColumn);
expect(Buffer.from(resolved[0].image as Uint8Array).toString()).toBe(
"file",
);
const images = resolved[0].images as [Uint8Array, { data: Uint8Array }];
expect(Buffer.from(images[0]).toString()).toBe("same");
expect(Buffer.from(images[1].data).toString()).toBe("same");
// Rows without Blob values and the caller's records are left alone.
expect(resolved[1]).toBe(data[1]);
expect(data[0].images?.[0]).toBe(shared);
});
it("reads each distinct Blob once, bare or inside data structs", async () => {
class CountingBlob extends Blob {
calls = 0;
async arrayBuffer(): Promise<ArrayBuffer> {
this.calls++;
return super.arrayBuffer();
}
}
const shared = new CountingBlob(["same"]);
const schema = new Schema([blob("image")]);
const resolved = await resolveBlobInputs(
[
{ image: { data: shared } },
{ image: { data: shared } },
{ image: shared },
],
schema,
);
expect(shared.calls).toBe(1);
const first = (resolved[0].image as { data: Uint8Array }).data;
expect(Buffer.from(first).toString()).toBe("same");
expect((resolved[1].image as { data: Uint8Array }).data).toBe(first);
expect(resolved[2].image).toBe(first);
});
it("reads a Blob held by a class instance", async () => {
const schema = new Schema([blob("image")]);
const [row] = await resolveBlobInputs(
[{ image: new BlobStruct(new Blob(["hi"])) }],
schema,
);
expect(
Buffer.from((row.image as { data: Uint8Array }).data).toString(),
).toBe("hi");
});
it("returns the input when there is nothing to read", async () => {
const data = [{ image: Buffer.from("x") }];
const schema = new Schema([blob("image")]);
await expect(resolveBlobInputs(data, schema)).resolves.toBe(data);
await expect(resolveBlobInputs(data)).resolves.toBe(data);
});
});
describe("convertToTable blob columns with embedding functions", () => {
// Embedding metadata routes rows through ensureNestedFieldsExist, which must
// leave blob values for the blob coercion instead of treating them as structs.
class LengthEmbedding extends EmbeddingFunction<string> {
ndims() {
return 2;
}
embeddingDataType(): Float {
return new Float32();
}
async computeSourceEmbeddings(data: string[]) {
return data.map((text) => [text.length, 1]);
}
}
getRegistry().register("blob-test-length")(LengthEmbedding);
const func = getRegistry().get<LengthEmbedding>("blob-test-length")!.create();
const embeddingSchema = LanceSchema({
text: func.sourceField(new Utf8()),
vector: func.vectorField(),
});
const schema = new Schema(
[...embeddingSchema.fields, blob("image")],
embeddingSchema.metadata,
);
it.each([
["Uint8Array", new Uint8Array([104, 105]), "hi", null],
["ArrayBuffer", new Uint8Array([104, 105]).buffer, "hi", null],
["Blob", new Blob(["hi"]), "hi", null],
["URI string", "s3://b/k", null, "s3://b/k"],
["URL", new URL("s3://b/k"), null, "s3://b/k"],
["data struct", { data: new Uint8Array([104, 105]) }, "hi", null],
])("accepts %s input", async (_name, image, data, uri) => {
const table = await convertToTable([{ text: "a", image }], undefined, {
schema,
});
const column = table.getChild("image")!;
expect(column.nullCount).toBe(0);
const bytes = column.getChild("data")!.get(0);
expect(bytes == null ? null : Buffer.from(bytes).toString()).toBe(data);
expect(column.getChild("uri")!.get(0)).toBe(uri);
});
it("writes a missing blob as null", async () => {
const table = await convertToTable([{ text: "a" }], undefined, { schema });
const column = table.getChild("image")!;
expect(column.nullCount).toBe(1);
expect(column.get(0)).toBeNull();
});
});
describe("BlobFile", () => {
it("rejects constructing BlobFile without a native handle", () => {
expect(() => new (BlobFile as unknown as { new (): BlobFile })()).toThrow(
/fetchBlobFiles/,
);
});
});
describe("makeArrowTable blob columns", () => {
it.each([
["ReadableStream", new ReadableStream()],
["Int16Array", new Int16Array([1])],
])("rejects %s input rather than writing a null blob", (_name, image) => {
const schema = new Schema([blob("image")]);
expect(() => makeArrowTable([{ image }], { schema })).toThrow(/image/);
});
it.each([
["Blob", new Blob(["hello"])],
["File", new File(["hello"], "hello.txt")],
["data struct with Blob", { data: new Blob(["hello"]) }],
[
"Blob with enumerable fields",
Object.assign(new Blob(["hello"]), { position: 0 }),
],
])("points %s input at the async APIs", (_name, image) => {
const schema = new Schema([blob("image")]);
expect(() => makeArrowTable([{ id: 1, image }], { schema })).toThrow(
/blob field image at row 0: Blob and File values must be read asynchronously/,
);
});
it.each([
["ArrayBuffer", new Uint8Array([104, 105]).buffer, "hi", null],
[
"data struct with ArrayBuffer",
{ data: new Uint8Array([104, 105]).buffer },
"hi",
null,
],
["URL", new URL("https://example.com/a"), null, "https://example.com/a"],
[
"uri struct with URL",
{ uri: new URL("file:///tmp/a.png") },
null,
"file:///tmp/a.png",
],
])("accepts %s input", (_name, image, data, uri) => {
const schema = new Schema([blob("image")]);
const input = { image };
const table = makeArrowTable([input], { schema });
const column = table.getChild("image")!;
expect(column.nullCount).toBe(0);
const bytes = column.getChild("data")!.get(0);
expect(bytes == null ? null : Buffer.from(bytes).toString()).toBe(data);
expect(column.getChild("uri")!.get(0)).toBe(uri);
// The caller's record is left untouched.
expect(input.image).toBe(image);
});
it.each([
["Uint8Array", new Uint8Array([104, 105]), "hi", null],
["URI string", "s3://b/k", null, "s3://b/k"],
["data struct", { data: new Uint8Array([104, 105]) }, "hi", null],
])("accepts %s input after a null row", (_name, image, data, uri) => {
const schema = new Schema([blob("image")]);
const table = makeArrowTable([{ image: null }, { image }], { schema });
const column = table.getChild("image")!;
expect(column.nullCount).toBe(1);
expect(column.get(0)).toBeNull();
const bytes = column.getChild("data")!.get(1);
expect(bytes == null ? null : Buffer.from(bytes).toString()).toBe(data);
expect(column.getChild("uri")!.get(1)).toBe(uri);
});
it("accepts a null struct before one holding a blob", () => {
const schema = new Schema([
new Field("info", new Struct([blob("image")]), true),
]);
const table = makeArrowTable(
[{ info: null }, { info: { image: new Uint8Array([97]) } }],
{ schema },
);
const info = table.getChild("info")!;
expect(info.get(0)).toBeNull();
const image = (info.get(1) as { image: { data: Uint8Array } }).image;
expect(Buffer.from(image.data).toString()).toBe("a");
});
it("skips a missing blob column named like an Object.prototype member", () => {
const schema = new Schema([
new Field("id", new Int32(), true),
blob("constructor"),
]);
const table = makeArrowTable([{ id: 1 }], { schema });
expect(table.schema.fields.map((f) => f.name)).toEqual(["id"]);
});
it("accepts ArrayBuffer and URL inside lists and structs", () => {
const schema = new Schema([
new Field("images", new List(blob("image")), true),
new Field("info", new Struct([blob("image")]), true),
]);
const table = makeArrowTable(
[
{
images: [new Uint8Array([97]).buffer, new URL("s3://b/k")],
info: { image: new Uint8Array([98]).buffer },
},
],
{ schema },
);
const images = Array.from(
table.getChild("images")!.get(0) as Iterable<{
data: Uint8Array | null;
uri: string | null;
}>,
);
expect(Buffer.from(images[0].data!).toString()).toBe("a");
expect(images[1].uri).toBe("s3://b/k");
const info = table.getChild("info")!.get(0) as {
image: { data: Uint8Array };
};
expect(Buffer.from(info.image.data).toString()).toBe("b");
});
it("coerces Buffer input onto a blob field", () => {
const schema = new Schema([
new Field("id", new Int64(), true),
blob("image"),
]);
const table = makeArrowTable([{ id: 1n, image: Buffer.from("hello") }], {
schema,
});
expect(isBlobField(table.schema.fields[1])).toBe(true);
const image = table.getChild("image")!;
expect(image.nullCount).toBe(0);
expect(image.getChild("uri")!.get(0)).toBeNull();
expect(image.getChild("data")!.nullCount).toBe(0);
expect(Buffer.from(image.getChild("data")!.get(0)!).toString()).toBe(
"hello",
);
});
it("coerces Buffer elements inside a list and keeps null slots", () => {
const schema = new Schema([
new Field("id", new Int64(), true),
new Field("images", new List(blob("image")), true),
]);
const table = makeArrowTable(
[
{ id: 1n, images: [Buffer.from("a"), Buffer.from("bb")] },
{ id: 2n, images: null },
{ id: 3n, images: [Buffer.from("c"), null] },
{ id: 4n, images: [] },
],
{ schema },
);
const images = table.getChild("images")!;
expect(images.nullCount).toBe(1);
const rows = images.toArray();
expect(rows[1]).toBeNull();
expect(Array.from(rows[3] as Iterable<unknown>)).toHaveLength(0);
const first = Array.from(rows[0] as Iterable<{ data: Uint8Array | null }>);
expect(Buffer.from(first[0].data!).toString()).toBe("a");
expect(Buffer.from(first[1].data!).toString()).toBe("bb");
const third = Array.from(
rows[2] as Iterable<{ data: Uint8Array | null } | null>,
);
expect(Buffer.from(third[0]!.data!).toString()).toBe("c");
expect(third[1]).toBeNull();
});
it("coerces Buffer fields inside list structs", () => {
const schema = new Schema([
new Field("id", new Int64(), true),
new Field(
"items",
new List(
new Field(
"item",
new Struct([new Field("name", new Utf8(), true), blob("image")]),
true,
),
),
true,
),
]);
const table = makeArrowTable(
[
{
id: 1n,
items: [{ name: "one", image: Buffer.from("alpha") }],
},
],
{ schema },
);
const items = Array.from(
table.getChild("items")!.toArray()[0] as Iterable<{
name: string;
image: { data: Uint8Array | null };
}>,
);
expect(items[0].name).toBe("one");
expect(Buffer.from(items[0].image.data!).toString()).toBe("alpha");
});
});
nodejs/__test__/catalog.test.ts
+142
View File
@@ -0,0 +1,142 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import * as http from "http";
import { Catalog, connectCatalog } from "../lancedb";
type RecordedRequest = {
url: string;
headers: http.IncomingHttpHeaders;
body: Record<string, unknown>;
};
async function withCatalog(
responses: [number, unknown][],
callback: (catalog: Catalog, requests: RecordedRequest[]) => Promise<void>,
) {
const requests: RecordedRequest[] = [];
const server = http.createServer(async (req, res) => {
const chunks: Buffer[] = [];
for await (const chunk of req) chunks.push(Buffer.from(chunk));
const body = Buffer.concat(chunks).toString();
requests.push({
url: req.url ?? "",
headers: req.headers,
body: body ? JSON.parse(body) : {},
});
const [status, response] = responses.shift() ?? [
500,
{ error: "Unexpected request" },
];
res.writeHead(status, { "content-type": "application/json" });
res.end(status === 204 ? undefined : JSON.stringify(response));
});
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
const address = server.address();
if (!address || typeof address === "string")
throw new Error("Missing server address");
try {
const catalog = await connectCatalog(`http://127.0.0.1:${address.port}`, {
apiKey: "secret",
sqlHostOverride: "grpc+tls://sql.example.com:10026",
clientConfig: {
extraHeaders: {
"X-LanceDB-Database": "wrong-static",
"X-LanceDB-Database-Prefix": "wrong",
},
},
headerProvider: () => ({
"X-LanceDB-Database": "wrong-dynamic",
"X-LanceDB-Database-Prefix": "wrong",
authorization: "Bearer refreshed",
}),
});
await callback(catalog, requests);
expect(responses).toHaveLength(0);
} finally {
server.closeAllConnections();
await new Promise<void>((resolve) => server.close(() => resolve()));
}
}
describe("remote catalog", () => {
it("uses root namespace routes and preserves independent database scope", async () => {
await withCatalog(
[
[204, null],
[200, { tables: [] }],
[200, {}],
[200, { tables: [] }],
[200, { tables: [] }],
// biome-ignore lint/style/useNamingConvention: server wire format
[200, { namespaces: ["team/search"], page_token: "next" }],
[204, null],
],
async (catalog, requests) => {
const first = await catalog.createDatabase("team/search", {
existOk: true,
});
expect(await first.tableNames()).toEqual([]);
const second = await catalog.connectDatabase("other");
expect(await second.tableNames()).toEqual([]);
expect(await first.tableNames()).toEqual([]);
expect(
await catalog.listDatabases({ limit: 1, pageToken: "a/b" }),
).toEqual({ databases: ["team/search"], pageToken: "next" });
await catalog.dropDatabase("team/search", { ignoreMissing: true });
expect(requests[0].url).toBe("/v1/namespace/team%2Fsearch/create");
expect(requests[0].body).toEqual({ mode: "ExistOk" });
expect(requests[5].url).toBe(
"/v1/namespace/$/list?limit=1&page_token=a%2Fb",
);
expect(requests[6].body).toEqual({
mode: "Skip",
behavior: "Restrict",
});
for (const [i, request] of requests.entries()) {
expect(request.headers["x-lancedb-database"]).toBe(
i === 1 || i === 4 ? "team/search" : i === 3 ? "other" : undefined,
);
expect(request.headers["x-lancedb-database-prefix"]).toBeUndefined();
expect(request.headers.authorization).toBe("Bearer refreshed");
}
},
);
});
it("propagates lifecycle errors and sends restricted drops", async () => {
await withCatalog(
[
[404, {}],
[409, {}],
[400, {}],
[404, {}],
],
async (catalog, requests) => {
await expect(catalog.connectDatabase("missing")).rejects.toThrow(
"missing",
);
await expect(catalog.createDatabase("exists")).rejects.toThrow(
"exists",
);
await expect(catalog.dropDatabase("full")).rejects.toThrow();
await catalog.dropDatabase("missing", { ignoreMissing: true });
expect(requests[2].body).toEqual({
mode: "Fail",
behavior: "Restrict",
});
},
);
});
it("validates endpoints and pagination", async () => {
await expect(connectCatalog("/tmp/catalog")).rejects.toThrow();
const catalog = await connectCatalog("http://127.0.0.1:1");
for (const limit of [0, -1, 1.5, 2147483648]) {
await expect(catalog.listDatabases({ limit })).rejects.toThrow("limit");
}
await expect(catalog.connectDatabase("a$b")).rejects.toThrow(
"Invalid database name",
);
});
});
nodejs/__test__/fixtures/oauth_browser.cmd
+3
View File
@@ -0,0 +1,3 @@
@rem SPDX-License-Identifier: Apache-2.0
@rem SPDX-FileCopyrightText: Copyright The LanceDB Authors
@exit /b 0
nodejs/__test__/materialized_view.test.ts
+66 -43
View File
@@ -28,6 +28,49 @@ describe("materialized views", () => {
});
afterEach(() => tmpDir.removeCallback());
it("reads stored queries and legacy layouts", () => {
const read = (stored: string) =>
definitionFromMetadata(new Map([[DEFINITION_META_KEY, stored]]), "v");
const query =
"SELECT id, c.chunk FROM ns.docs, UNNEST(chunks) AS c WHERE id > 1";
expect(read(`{"format":1,"query":${JSON.stringify(query)}}`).query).toBe(
query,
);
// The structured layout written before the format number reads as the
// query it described, under either of its kind tags.
expect(
read(
'{"kind":"namespaced_select","source_table":"people","source_namespace":["ns"],' +
'"projections":[{"output":"name","expression":"`name`"},' +
'{"output":"Shout","expression":"upper(name)"}],"filter":"age >= 18","limit":42}',
).query,
).toBe(
"SELECT `name`, upper(name) AS `Shout` FROM ns.people WHERE age >= 18 LIMIT 42",
);
expect(read('{"kind":"select","source_table":"people"}').query).toBe(
"SELECT * FROM people",
);
for (const [format, fn] of [
[3, "vector_duplicate_pairs"],
[4, "vector_dedup"],
]) {
const native = `SELECT * FROM ${fn}('images', 3, 'phash', 4)`;
expect(read(JSON.stringify({ format, query: native })).query).toBe(
native,
);
}
// A newer writer's layout is reported, never guessed at.
for (const newer of [
`{"format":5,"query":${JSON.stringify(query)}}`,
'{"kind":"select_v3","source_table":"people"}',
]) {
expect(() => read(newer)).toThrow(/cannot refresh/);
}
});
it("rejects a stored limit a number cannot carry", () => {
const big = new Map([
[
@@ -38,36 +81,6 @@ describe("materialized views", () => {
expect(() => definitionFromMetadata(big, "v")).toThrow(
/too large to represent exactly/,
);
const safe = new Map([
[
DEFINITION_META_KEY,
'{"kind":"select","source_table":"people","limit":42}',
],
]);
expect(definitionFromMetadata(safe, "v").limit).toBe(42);
});
it("reads the namespaced select kind and refuses unknown kinds", () => {
// "namespaced_select" is the namespaced form of "select": same shape, a
// separate kind so readers that predate it refuse instead of resolving
// the source at the root.
const namespaced = new Map([
[
DEFINITION_META_KEY,
'{"kind":"namespaced_select","source_table":"people","source_namespace":["ns"]}',
],
]);
const definition = definitionFromMetadata(namespaced, "v");
expect(definition.sourceTable).toBe("people");
expect(definition.sourceNamespace).toEqual(["ns"]);
const unknown = new Map([
[DEFINITION_META_KEY, '{"kind":"select_v3","source_table":"people"}'],
]);
expect(() => definitionFromMetadata(unknown, "v")).toThrow(
/cannot refresh/,
);
});
it("creates, refreshes and queries a view", async () => {
@@ -76,11 +89,7 @@ describe("materialized views", () => {
where: "age >= 18",
});
expect(view.name).toBe("adults");
expect(await view.table().countRows()).toBe(0);
const result = await view.refresh();
expect(result.mode).toBe("rebuild");
expect(Number(result.rowsWritten)).toBe(2);
expect(await view.table().countRows()).toBe(2);
const rows = await view.table().query().toArray();
expect(rows.map((r) => r.shout).sort()).toEqual(["ADA", "GRACE"]);
@@ -92,17 +101,15 @@ describe("materialized views", () => {
});
const view = await db.openMaterializedView("adults");
const definition = await view.definition();
expect(definition.sourceTable).toBe("people");
expect(definition.filter).toBe("age >= 18");
expect(definition.projections).toEqual([
["name", "`name`"],
["age", "`age`"],
]);
expect(definition.inputs).toEqual(["age", "name"]);
expect(definition.query).toBe(
"SELECT name, age FROM people WHERE age >= 18",
);
});
it("refreshes incrementally after an append", async () => {
const view = await db.createMaterializedView("copy", "people");
const view = await db.createMaterializedView("copy", "people", {
withNoData: true,
});
await view.refresh();
const people = await db.openTable("people");
@@ -123,6 +130,21 @@ describe("materialized views", () => {
await expect(db.openMaterializedView("people")).rejects.toThrow(
"not a materialized view",
);
await expect(db.dropMaterializedView("people")).rejects.toThrow(
"not a materialized view",
);
await db.dropMaterializedView("adults");
expect(await db.listMaterializedViews()).toEqual([]);
});
it("returns a job when dropping a view asynchronously", async () => {
await db.createMaterializedView("adults", "people");
const job = await db.dropMaterializedViewAsync("adults");
expect(job.id).toBeNull();
await job.wait();
expect(await db.listMaterializedViews()).toEqual([]);
});
it("rejects an invalid expression at create time", async () => {
@@ -155,6 +177,7 @@ describe("materialized views", () => {
});
const view = await db.createMaterializedView("quoted", "odd_names", {
select: ["order item"],
withNoData: true,
});
const result = await view.refresh();
expect(Number(result.rowsWritten)).toBe(1);
nodejs/__test__/oauth.test.ts
+245
View File
@@ -0,0 +1,245 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import * as fs from "fs";
import * as http from "http";
import { execFileSync } from "node:child_process";
import * as os from "os";
import * as path from "path";
import { OAuthConfig, OAuthFlowType, OAuthSession } from "../lancedb/oauth";
function tempCacheDir(): string {
return fs.mkdtempSync(path.join(os.tmpdir(), "lancedb-oauth-cache-"));
}
function deviceConfig(issuerUrl: string, cacheDir: string): OAuthConfig {
return {
issuerUrl,
clientId: "client-id",
scopes: ["openid"],
flow: OAuthFlowType.DeviceCode,
tokenCache: { cacheDir },
};
}
describe("OAuthSession", () => {
beforeAll(() => {
// Child processes inherit the real environment, just as Rust reads it.
// Fail before login if the browser override only exists in Jest's sandbox.
const browser = execFileSync(
process.execPath,
["-p", "process.env.LANCEDB_OAUTH_BROWSER ?? ''"],
{ encoding: "utf8" },
).trim();
expect(browser).not.toBe("");
expect(browser).toBe(process.env.LANCEDB_OAUTH_BROWSER);
});
it("reports an absent session and logout is idempotent", async () => {
const cacheDir = tempCacheDir();
const session = new OAuthSession(
deviceConfig("https://issuer.example.com", cacheDir),
);
const status = await session.status();
expect(status.refreshable).toBe(false);
expect(status.issuerUrl).toBe("https://issuer.example.com");
expect(status.clientId).toBe("client-id");
expect(status.scopes).toEqual(["openid"]);
expect(status.flow).toBe("device_code");
expect(status.obtainedAt).toBeUndefined();
const logout = await session.logout();
expect(logout.removed).toBe(false);
});
it("requires token cache options", () => {
const config: OAuthConfig = {
issuerUrl: "https://issuer.example.com",
clientId: "client-id",
scopes: ["openid"],
flow: OAuthFlowType.DeviceCode,
};
expect(() => new OAuthSession(config)).toThrow(/token/);
});
it("rejects azure managed identity persistence", () => {
const config: OAuthConfig = {
issuerUrl: "https://login.microsoftonline.com/tenant/v2.0",
clientId: "app-id",
scopes: ["api://app/.default"],
flow: OAuthFlowType.AzureManagedIdentity,
tokenCache: { cacheDir: tempCacheDir() },
};
expect(() => new OAuthSession(config)).toThrow(/AzureManagedIdentity/);
});
it.each([
{},
{
resource: "https://api.example.com/a?x=1&y=two",
audience: "audience + & / ü",
},
])(
"logs in via device flow with target %j, caches, and logs out",
async (target) => {
const server = new MockIdp();
await server.start();
try {
const cacheDir = tempCacheDir();
const issuerUrl = server.issuerUrl();
const config = { ...deviceConfig(issuerUrl, cacheDir), ...target };
const session = new OAuthSession(config);
const status = await session.login();
expect(status.refreshable).toBe(true);
expect(status.resource).toBe(config.resource);
expect(status.audience).toBe(config.audience);
expect(status.obtainedAt).toBeGreaterThan(0);
expect(server.state.deviceAuthorizations).toBe(1);
// An independent session (a fresh "process") sees the cached login.
const other = new OAuthSession(config);
const cached = await other.status();
expect(cached.refreshable).toBe(true);
const logout = await other.logout();
expect(logout.removed).toBe(true);
const again = await session.logout();
expect(again.removed).toBe(false);
expect((await session.status()).refreshable).toBe(false);
// Only the initial login used the interactive device flow.
expect(server.state.deviceAuthorizations).toBe(1);
expect(server.state.refreshGrants).toBe(0);
expect(server.requests).toHaveLength(2);
for (const params of server.requests) {
expect(params.getAll("resource")).toEqual(
config.resource === undefined ? [] : [config.resource],
);
expect(params.getAll("audience")).toEqual(
config.audience === undefined ? [] : [config.audience],
);
}
} finally {
server.close();
}
},
15000,
);
});
/** Mock IdP with discovery, device authorization, and rotating refresh. */
class MockIdp {
readonly requests: URLSearchParams[] = [];
readonly state = {
deviceAuthorizations: 0,
refreshGrants: 0,
accessTokensIssued: 0,
currentRefresh: null as string | null,
};
private server?: http.Server;
private port = 0;
issuerUrl(): string {
return `http://127.0.0.1:${this.port}`;
}
async start(): Promise<void> {
const server = http.createServer((req, res) => {
const chunks: Buffer[] = [];
req.on("data", (chunk) => chunks.push(chunk));
req.on("end", () => {
const body = Buffer.concat(chunks).toString();
const params = new URLSearchParams(body);
this.handle(req.url ?? "", params, res);
});
});
await new Promise<void>((resolve) => {
server.listen(0, "127.0.0.1", () => resolve());
});
const address = server.address();
if (address && typeof address === "object") {
this.port = address.port;
}
this.server = server;
}
private handle(
url: string,
params: URLSearchParams,
res: http.ServerResponse,
): void {
const respond = (status: number, payload: unknown): void => {
const body = JSON.stringify(payload);
res.writeHead(status, {
"Content-Type": "application/json",
"Content-Length": Buffer.byteLength(body),
});
res.end(body);
};
if (url === "/.well-known/openid-configuration") {
respond(200, {
// biome-ignore lint/style/useNamingConvention: OAuth wire format
token_endpoint: `${this.issuerUrl()}/token`,
// biome-ignore lint/style/useNamingConvention: OAuth wire format
device_authorization_endpoint: `${this.issuerUrl()}/device`,
});
return;
}
if (url === "/device" || url === "/token") {
this.requests.push(params);
}
if (url === "/device") {
this.state.deviceAuthorizations += 1;
respond(200, {
// biome-ignore lint/style/useNamingConvention: OAuth wire format
device_code: "device-code",
// biome-ignore lint/style/useNamingConvention: OAuth wire format
user_code: "ABCD-EFGH",
// biome-ignore lint/style/useNamingConvention: OAuth wire format
verification_uri: `${this.issuerUrl()}/verify`,
// biome-ignore lint/style/useNamingConvention: OAuth wire format
expires_in: 60,
interval: 1,
});
return;
}
if (url === "/token") {
const grantType = params.get("grant_type") ?? "";
if (grantType === "refresh_token") {
this.state.refreshGrants += 1;
if (params.get("refresh_token") !== this.state.currentRefresh) {
respond(400, { error: "invalid_grant" });
return;
}
} else if (!grantType.includes("device_code")) {
respond(400, { error: "unsupported_grant_type" });
return;
}
this.state.accessTokensIssued += 1;
const number = this.state.accessTokensIssued;
const refresh = `refresh-${number}`;
this.state.currentRefresh = refresh;
respond(200, {
// biome-ignore lint/style/useNamingConvention: OAuth wire format
access_token: `access-${number}`,
// biome-ignore lint/style/useNamingConvention: OAuth wire format
refresh_token: refresh,
// biome-ignore lint/style/useNamingConvention: OAuth wire format
expires_in: 3600,
});
return;
}
respond(404, {});
}
close(): void {
this.server?.close();
}
}
nodejs/__test__/query.test.ts
+123
View File
@@ -110,6 +110,129 @@ describe("Query outputSchema", () => {
});
});
describe("Query empty results", () => {
let tmpDir: tmp.DirResult;
let table: Table;
beforeEach(async () => {
tmpDir = tmp.dirSync({ unsafeCleanup: true });
const db = await connect(tmpDir.name);
const schema = new Schema(
[
new Field("id", new Int64(), false),
new Field(
"text",
new Utf8(),
true,
new Map([["description", "searchable text"]]),
),
new Field(
"vector",
new FixedSizeList(2, new Field("item", new Float32())),
true,
),
],
new Map([["source", "empty-query-regression"]]),
);
table = await db.createTable(
"test",
makeArrowTable([{ id: 1n, text: "hello", vector: [1, 2] }], { schema }),
);
});
afterEach(() => {
tmpDir.removeCallback();
});
it.each([
{
name: "filtered scan",
query: (table: Table) => table.query().where("id < 0"),
fields: ["id", "text", "vector"],
},
{
name: "zero limit",
query: (table: Table) => table.query().limit(0),
fields: ["id", "text", "vector"],
},
{
name: "column projection",
query: (table: Table) =>
table.query().where("id < 0").select(["text", "id"]),
fields: ["text", "id"],
},
{
name: "dynamic projection",
query: (table: Table) =>
table.query().where("id < 0").select({ doubled: "id * 2" }),
fields: ["doubled"],
},
{
name: "filtered vector search",
query: (table: Table) =>
table.search([0.5, 0.5]).where("id < 0").limit(3),
fields: ["id", "text", "vector", "_distance"],
},
{
name: "vector projection with row id",
query: (table: Table) =>
table
.vectorSearch([0.5, 0.5])
.where("id < 0")
.select(["id"])
.withRowId(),
fields: ["id", "_distance", "_rowid"],
},
{
name: "fast search without an index",
query: (table: Table) => table.search([0.5, 0.5]).fastSearch(),
fields: ["id", "text", "vector", "_rowid", "_distance"],
},
{
name: "take with no offsets",
query: (table: Table) => table.takeOffsets([]).select(["text"]),
fields: ["text"],
},
])("preserves the output schema for $name", async ({ query, fields }) => {
const result = await query(table).toArrow({ maxBatchLength: 1 });
expect(result.numRows).toBe(0);
expect(result.schema.fields.map((field) => field.name)).toEqual(fields);
for (const name of fields) {
expect(result.getChild(name)).not.toBeNull();
expect(result.getChild(name)?.length).toBe(0);
}
});
it("preserves types, nullability, and metadata", async () => {
const result = await table.query().where("id < 0").toArrow();
const schema = await table.schema();
expect(result.schema).toEqual(schema);
expect(result.getChild("id")?.type.toString()).toBe("Int64");
expect(result.schema.fields[0].nullable).toBe(false);
expect(result.schema.fields[1].metadata.get("description")).toBe(
"searchable text",
);
expect(result.schema.metadata.get("source")).toBe("empty-query-regression");
});
it("preserves the score column for full-text search with no matches", async () => {
await table.createIndex("text", { config: Index.fts() });
const result = await table
.search("missing", "fts")
.select(["id"])
.toArrow();
expect(result.numRows).toBe(0);
expect(result.schema.fields.map((field) => field.name)).toEqual([
"id",
"_score",
]);
expect(result.getChild("_score")?.type.toString()).toBe("Float32");
});
});
describe("Search pagination", () => {
let tmpDir: tmp.DirResult;
let table: Table;
nodejs/__test__/registry.test.ts
+5 -8
View File
@@ -3,13 +3,10 @@
import * as apiArrow from "apache-arrow";
import * as arrow15 from "apache-arrow-15";
import * as arrow16 from "apache-arrow-16";
import * as arrow17 from "apache-arrow-17";
import * as arrow18 from "apache-arrow-18";
import * as tmp from "tmp";
import { arrowVersions } from "./arrow_versions";
import { connect } from "../lancedb";
import {
EmbeddingFunction,
@@ -18,7 +15,7 @@ import {
} from "../lancedb/embedding";
import { getRegistry, register } from "../lancedb/embedding/registry";
describe.each([arrow15, arrow16, arrow17, arrow18])("LanceSchema", (arrow) => {
describe.each(arrowVersions)("LanceSchema", (arrow) => {
test("should preserve input order", async () => {
const schema = LanceSchema({
id: new arrow.Int32(),
@@ -29,7 +26,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])("LanceSchema", (arrow) => {
});
});
describe.each([arrow15, arrow16, arrow17, arrow18])("Registry", (arrow) => {
describe.each(arrowVersions)("Registry", (arrow) => {
let tmpDir: tmp.DirResult;
beforeEach(() => {
tmpDir = tmp.dirSync({ unsafeCleanup: true });
@@ -240,7 +237,7 @@ describe("Registry.setVar", () => {
return data.map(() => [1, 2, 3]);
}
embeddingDataType() {
return new arrow18.Float32() as apiArrow.Float;
return new apiArrow.Float32();
}
protected getSensitiveKeys() {
return ["secretKey"];
nodejs/__test__/remote.test.ts
+444 -20
View File
@@ -5,12 +5,23 @@ import * as http from "http";
import { RequestListener } from "http";
import packageJson = require("../package.json");
import {
ClientAuthMethod,
ClientConfig,
Connection,
ConnectionOptions,
OAuthConfig,
OAuthFlowType,
TlsConfig,
connect,
} from "../lancedb";
import {
Table as ArrowTable,
Field,
Int64,
RecordBatch,
Schema,
tableToIPC,
} from "../lancedb/arrow";
import {
HeaderProvider,
OAuthHeaderProvider,
@@ -82,21 +93,264 @@ async function withMockDatabase(
}
describe("remote connection", () => {
it("refuses materialized views before issuing any request", async () => {
const paths: string[] = [];
it("rejects nprobes(0) before sending a query", async () => {
const requests: string[] = [];
await withMockDatabase(
(req, res) => {
paths.push(req.url ?? "");
requests.push(req.url ?? "");
if (req.url === "/v1/table/test/describe/") {
res
.writeHead(200, { "Content-Type": "application/json" })
.end(JSON.stringify({ version: 1, schema: { fields: [] } }));
} else {
res.writeHead(404).end();
}
},
async (db) => {
const table = await db.openTable("test");
expect(() => table.vectorSearch([0, 0]).nprobes(0)).toThrow(
"Invalid input, nprobes must be greater than 0",
);
expect(() =>
table.query().nearestTo([0, 0]).fullTextSearch("dog").nprobes(0),
).toThrow("Invalid input, nprobes must be greater than 0");
await expect(
table
.vectorSearch(Promise.resolve([0, 0]))
.nprobes(0)
.toArrow(),
).rejects.toThrow("Invalid input, nprobes must be greater than 0");
},
);
expect(requests).toEqual(["/v1/table/test/describe/"]);
});
it.each([false, true])(
"preserves an empty query's schema with an empty batch: %s",
async (withEmptyBatch) => {
const schema = new Schema(
[new Field("doubled", new Int64(), false)],
new Map([["source", "query-output"]]),
);
const result = new ArrowTable(
schema,
withEmptyBatch ? [new RecordBatch(schema, undefined)] : [],
);
const response = Buffer.from(tableToIPC(result, "stream"));
let queryRequests = 0;
await withMockDatabase(
(req, res) => {
if (req.url?.endsWith("/describe/")) {
res.writeHead(200, { "Content-Type": "application/json" }).end(
JSON.stringify({
name: "items",
version: 1,
schema: {
fields: [
{ name: "id", type: { type: "int64" }, nullable: false },
],
},
}),
);
} else if (req.url?.endsWith("/query/")) {
queryRequests++;
req.resume();
req.on("end", () => {
res
.writeHead(200, {
"Content-Type": "application/vnd.apache.arrow.stream",
})
.end(response);
});
} else {
res.writeHead(404).end();
}
},
async (db) => {
const table = await db.openTable("items");
const result = await table
.query()
.where("id < 0")
.select({ doubled: "id * 2" })
.toArrow();
expect(result.numRows).toBe(0);
expect(result.schema).toEqual(schema);
expect(result.getChild("doubled")?.length).toBe(0);
},
);
expect(queryRequests).toBe(1);
},
);
it("rejects the external blob opt-in without blocking regular adds", async () => {
let insertRequests = 0;
const describeRequests: string[] = [];
await withMockDatabase(
(req, res) => {
const requestPath = req.url ?? "";
if (requestPath.endsWith("/describe/")) {
describeRequests.push(requestPath);
res.writeHead(200, { "Content-Type": "application/json" }).end(
JSON.stringify({
name: "items",
version: 1,
schema: {
fields: [
{
name: "id",
type: { type: "int64" },
nullable: true,
},
],
},
}),
);
return;
}
if (requestPath.endsWith("/insert/")) {
insertRequests++;
req.resume();
req.on("end", () => {
res
.writeHead(200, { "Content-Type": "application/json" })
.end(JSON.stringify({ version: insertRequests + 1 }));
});
return;
}
res.writeHead(404).end();
},
async (db) => {
await expect(db.openMaterializedView("secret_table")).rejects.toThrow(
/only on local databases/,
const table = await db.openTable("items");
await expect(
table.add([{ id: 1n }], {
allowExternalBlobOutsideBases: true,
}),
).rejects.toThrow("only supported on local tables");
expect(insertRequests).toBe(0);
await expect(table.add([{ id: 2n }])).resolves.toMatchObject({
version: 2,
});
await expect(
table.add([{ id: 3n }], {
allowExternalBlobOutsideBases: false,
}),
).resolves.toMatchObject({ version: 3 });
},
);
expect(describeRequests.length).toBeGreaterThan(0);
expect(insertRequests).toBe(2);
});
it("lists materialized views through the namespace route", async () => {
await withMockDatabase(
(req, res) => {
expect(req.method).toBe("GET");
expect(req.url).toBe("/v1/namespace/$/materialized_view/list");
res
.writeHead(200, { "content-type": "application/json" })
.end(JSON.stringify({ views: ["daily_sales"] }));
},
async (db) => {
expect(await db.listMaterializedViews()).toEqual(["daily_sales"]);
},
);
});
it("creates a view and decodes the schema it resolved to", async () => {
await withMockDatabase(
(req, res) => {
expect(req.method).toBe("POST");
expect(req.url).toBe("/v1/view/analytics$adults/create");
res.writeHead(200, { "content-type": "application/json" }).end(
JSON.stringify({
name: "adults",
namespace: ["analytics"],
query: "SELECT name FROM people",
// biome-ignore lint/style/useNamingConvention: the wire field is snake_case
default_database: "db",
// biome-ignore lint/style/useNamingConvention: the wire field is snake_case
default_namespace: ["analytics"],
schema: {
fields: [
{ name: "name", nullable: true, type: { type: "utf8" } },
],
},
}),
);
await expect(db.listMaterializedViews()).rejects.toThrow(
/only on local databases/,
);
expect(paths).toEqual([]);
},
async (db) => {
const view = await db.createView("adults", "SELECT name FROM people", [
"analytics",
]);
expect(view.name).toBe("adults");
expect(view.namespacePath).toEqual(["analytics"]);
expect(view.query).toBe("SELECT name FROM people");
expect(view.defaultDatabase).toBe("db");
expect(view.defaultNamespacePath).toEqual(["analytics"]);
expect(view.schema.fields.map((f) => f.name)).toEqual(["name"]);
},
);
});
it("lists and drops views through their own routes", async () => {
await withMockDatabase(
(req, res) => {
expect(req.url).toBe("/v1/namespace/$/view/list");
res
.writeHead(200, { "content-type": "application/json" })
.end(JSON.stringify({ views: ["adults"] }));
},
async (db) => {
expect(await db.listViews()).toEqual(["adults"]);
},
);
await withMockDatabase(
(req, res) => {
expect(req.method).toBe("POST");
expect(req.url).toBe("/v1/view/adults/drop");
res.writeHead(200, { "content-type": "application/json" }).end("{}");
},
async (db) => {
await db.dropView("adults");
},
);
});
it("reports the cleanup job when a view drop is accepted", async () => {
await withMockDatabase(
(req, res) => {
expect(req.method).toBe("POST");
expect(req.url).toBe("/v1/view/adults/drop");
res
.writeHead(202, { "content-type": "application/json" })
.end('{"job_id": "j1-do-abc"}');
},
async (db) => {
const job = await db.dropViewAsync("adults");
expect(job.id).toBe("j1-do-abc");
},
);
});
it("reports a finished job when a view drop had nothing to delete", async () => {
await withMockDatabase(
(req, res) => {
expect(req.url).toBe("/v1/view/adults/drop");
res.writeHead(200, { "content-type": "application/json" }).end("{}");
},
async (db) => {
// A 200 means the name was not bound, so there is no cleanup to wait on.
const job = await db.dropViewAsync("adults");
expect(job.id).toBeNull();
await job.wait();
},
);
});
@@ -196,6 +450,66 @@ describe("remote connection", () => {
);
});
it("lists the rows a Function refresh skipped", async () => {
const bodies: unknown[] = [];
await withMockDatabase(
(req, res) => {
const path = req.url ?? "";
if (path.endsWith("/describe/")) {
res.writeHead(200, { "Content-Type": "application/json" }).end(
JSON.stringify({
name: "docs",
version: 1,
schema: { fields: [] },
}),
);
return;
}
if (path === "/v1/table/docs/errors") {
let body = "";
req.on("data", (chunk) => {
body += chunk;
});
req.on("end", () => {
bodies.push(JSON.parse(body));
res.writeHead(200, { "Content-Type": "application/json" }).end(
`{"records": [{"job_id": "j-7", "fragment_id": 3, "row_offset": 9,
"column": "embedding", "function": "embed", "function_version": "2",
"table_version": 11, "error_type": "ValueError",
"error_message": "bad input 'x'", "created_at_millis": 1700000000000}],
"fragments": [{"job_id": "j-7", "fragment_id": 4, "rows_skipped": 500,
"rows_recorded": 100}], "truncated": true}`,
);
});
return;
}
res.writeHead(404).end();
},
async (db) => {
const table = await db.openTable("docs");
const errors = await table.functionErrors({
jobId: "j-7",
column: "embedding",
limit: 2,
});
expect(errors.truncated).toBe(true);
expect(errors.records.map((r) => r.errorMessage)).toEqual([
"bad input 'x'",
]);
expect(errors.records[0].rowOffset).toBe(9);
expect(errors.fragments[0].rowsSkipped).toBe(500);
await table.functionErrors();
await expect(table.functionErrors({ limit: -1 })).rejects.toThrow(
"limit must be a non-negative integer",
);
},
);
expect(bodies).toEqual([
JSON.parse('{"job_id": "j-7", "column": "embedding", "limit": 2}'),
{},
]);
});
it("surfaces JSON server errors from remote table operations", async () => {
await withMockDatabase(
(req, res) => {
@@ -442,6 +756,40 @@ describe("remote connection", () => {
]);
});
describe("OAuthConfig", () => {
it("should expose client auth method values", () => {
expect(ClientAuthMethod.None).toBe("none");
expect(ClientAuthMethod.ClientSecretBasic).toBe("client_secret_basic");
expect(ClientAuthMethod.ClientSecretPost).toBe("client_secret_post");
});
it("should accept a confidential client with basic auth", () => {
const config: OAuthConfig = {
issuerUrl: "https://issuer.example.com",
clientId: "client-id",
clientSecret: "secret",
scopes: ["openid"],
flow: OAuthFlowType.AuthorizationCode,
clientAuthMethod: ClientAuthMethod.ClientSecretBasic,
};
expect(config.clientAuthMethod).toBe(ClientAuthMethod.ClientSecretBasic);
});
it("should accept a public PKCE client without auth method or secret", () => {
const config: OAuthConfig = {
issuerUrl: "https://issuer.example.com",
clientId: "client-id",
scopes: ["openid"],
flow: OAuthFlowType.AuthorizationCode,
usePkce: true,
};
expect(config.clientSecret).toBeUndefined();
expect(config.clientAuthMethod).toBeUndefined();
});
});
describe("TlsConfig", () => {
it("should create TlsConfig with all fields", () => {
const tlsConfig: TlsConfig = {
@@ -939,6 +1287,7 @@ describe("remote connection jobs surface", () => {
const { tableFromArrays, tableToIPC } = await import("apache-arrow");
const eventsTable = tableFromArrays({ state: ["created", "succeeded"] });
const eventsBody = Buffer.from(tableToIPC(eventsTable, "stream"));
const queryEventsPayloads: Record<string, unknown>[] = [];
await withMockDatabase(
(req, res) => {
@@ -967,6 +1316,16 @@ describe("remote connection jobs surface", () => {
);
}
} else if (req.url === "/v1/jobs/describe") {
if (payload["job_id"] === "job-2") {
res
.writeHead(200, { "Content-Type": "application/json" })
.end(
'{"job_id": "job-2", "job_type": "refresh_column", ' +
'"job_state": "DONE", "creation_ms": 2000, ' +
'"result": {"rows_assigned": 1000000}}',
);
return;
}
if (payload["job_id"] !== "job-1") {
res.writeHead(404).end("no such job");
return;
@@ -987,7 +1346,26 @@ describe("remote connection jobs surface", () => {
res
.writeHead(200, { "Content-Type": "application/json" })
.end('{"job_id": "job-1"}');
} else if (req.url === "/v1/jobs/pause") {
if (payload["job_id"] !== "job-1") {
res.writeHead(404).end("no such job");
return;
}
res
.writeHead(200, { "Content-Type": "application/json" })
.end('{"job_id": "job-1", "paused": true}');
} else if (req.url === "/v1/jobs/resume") {
if (payload["job_id"] !== "job-1") {
res.writeHead(404).end("no such job");
return;
}
res
.writeHead(200, { "Content-Type": "application/json" })
.end(
'{"job_id": "job-1", "resumed": false, "still_pausing": true}',
);
} else if (req.url === "/v1/jobs/query_events") {
queryEventsPayloads.push(payload);
res
.writeHead(200, {
"Content-Type": "application/vnd.apache.arrow.stream",
@@ -1004,22 +1382,68 @@ describe("remote connection jobs surface", () => {
expect(jobs[0].state).toEqual("running");
expect(jobs[1].state).toEqual("finished");
const description = await db.getJob("job-1");
expect(description?.state).toEqual("failed");
expect(JSON.parse(description?.specJson ?? "")).toEqual({
column: "vec",
});
expect(description?.failure?.message).toEqual("worker died");
expect(await db.getJob("missing")).toBeNull();
expect(await db.cancelJob("job-1")).toBe(true);
expect(await db.cancelJob("missing")).toBe(false);
const history = await db.jobHistory("job-1");
expect(history.numRows).toEqual(2);
expect(await db.pauseJob("job-1")).toEqual("pausing");
expect(await db.resumeJob("job-1")).toEqual("still_pausing");
const job = db.job("job-1");
// Opening a job hands back a populated handle; a missing one rejects.
await expect(db.openJob("missing")).rejects.toThrow("not found");
const finished = await db.openJob("job-2");
expect(finished.state).toEqual("finished");
expect(finished.result).toEqual({
// biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
rows_assigned: 1000000,
});
const job = await db.openJob("job-1");
expect(job.id).toEqual("job-1");
// openJob already populated the handle; refresh() re-reads it.
expect(job.state).toEqual("failed");
await job.refresh();
expect(job.state).toEqual("failed");
expect(job.jobType).toEqual("create_index");
expect(job.creationMs).toEqual(1000);
expect(job.spec).toEqual({ column: "vec" });
expect(job.result).toBeNull();
expect(job.failure?.message).toEqual("worker died");
// The handle reaches its own events, supplying its job id.
const jobEvents = await job.events({
limit: 500,
filter: "state = 'claim_complete'",
});
expect(jobEvents.numRows).toEqual(2);
expect(queryEventsPayloads.pop()).toEqual({
// biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
job_id: "job-1",
limit: 500,
filter: "state = 'claim_complete'",
});
// Printing lays every known field out on its own line, with the JSON
// payloads indented rather than crammed onto one line.
expect(`${job}`).toEqual(
[
"Job(",
' id="job-1",',
' state="failed",',
' jobType="create_index",',
" creationMs=1000,",
" spec={",
' "column": "vec"',
" },",
" failure={",
' "phase": "execute",',
' "message": "worker died",',
' "retryable": true',
" },",
")",
].join("\n"),
);
expect(await job.status()).toEqual("failed");
await expect(job.wait()).rejects.toThrow("worker died");
},
nodejs/__test__/sanitize.test.ts
+113 -1
View File
@@ -2,7 +2,13 @@
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import * as arrow from "../lancedb/arrow";
import { sanitizeField, sanitizeMap, sanitizeType } from "../lancedb/sanitize";
import {
sanitizeField,
sanitizeMap,
sanitizeSchema,
sanitizeType,
} from "../lancedb/sanitize";
import { arrow21 } from "./arrow_versions";
describe("sanitize", function () {
describe("sanitizeType function", function () {
@@ -59,6 +65,8 @@ describe("sanitize", function () {
["durationmicrosecond", new arrow.DurationMicrosecond()],
["durationmillisecond", new arrow.DurationMillisecond()],
["durationsecond", new arrow.DurationSecond()],
["largebinary", new arrow.LargeBinary()],
["largeutf8", new arrow.LargeUtf8()],
] as const;
it.each(allTypeNameTestCases)(
@@ -89,6 +97,110 @@ describe("sanitize", function () {
"Unrecognized type name in schema: invalid_type",
);
});
// These types only exist in Arrow 20 (IntervalMonthDayNano) and 21 (the
// rest), so the objects come from a second copy of the newest release. The
// library itself may be running on an older Arrow (see the CI matrix), in
// which case sanitizing them has to fail with a clear message instead.
const hostArrow = arrow as unknown as Record<string, unknown>;
const hostSupportsNewerTypes = typeof hostArrow.Utf8View === "function";
const whenSupported = hostSupportsNewerTypes ? describe : describe.skip;
const whenUnsupported = hostSupportsNewerTypes ? describe.skip : describe;
const newerTypeTestCases = [
["Utf8View", () => new arrow21.Utf8View()],
["BinaryView", () => new arrow21.BinaryView()],
["IntervalMonthDayNano", () => new arrow21.IntervalMonthDayNano()],
[
"LargeList",
() =>
new arrow21.LargeList(new arrow21.Field("item", new arrow21.Utf8())),
],
] as const;
whenSupported("types added in Arrow 20 and 21", function () {
it.each(newerTypeTestCases)(
"should sanitize a %s type from another copy of Arrow",
function (name, makeForeign) {
const foreign = makeForeign();
const result = sanitizeType(foreign);
// Interval subclasses share `Type.Interval`, so the sanitized type
// is the generic `Interval` carrying the same unit, as for the
// other units.
const expected = name === "IntervalMonthDayNano" ? "Interval" : name;
expect(result).toBeInstanceOf(hostArrow[expected] as () => unknown);
expect(result.typeId).toBe(foreign.typeId);
expect(result.toString()).toBe(foreign.toString());
},
);
it.each([
["binaryview", "BinaryView"],
["utf8view", "Utf8View"],
["intervalmonthdaynano", "Interval"],
])('should map type name "%s" to %s', function (name, expected) {
expect(sanitizeType(name)).toBeInstanceOf(
hostArrow[expected] as () => unknown,
);
});
it("should sanitize the child field of a LargeList", function () {
const foreign = new arrow21.LargeList(
new arrow21.Field("item", new arrow21.Int32(), false),
);
const result = sanitizeType(foreign) as arrow.List;
expect(result.children[0]).toBeInstanceOf(arrow.Field);
expect(result.children[0].name).toBe("item");
expect(result.children[0].nullable).toBe(false);
const child = result.children[0].type;
expect(arrow.DataType.isInt(child)).toBe(true);
expect((child as arrow.Int).bitWidth).toBe(32);
expect((child as arrow.Int).isSigned).toBe(true);
});
});
whenUnsupported("types newer than the installed Arrow", function () {
it.each(
newerTypeTestCases.filter(([name]) => name !== "IntervalMonthDayNano"),
)(
"should explain that %s needs a newer apache-arrow",
function (name, makeForeign) {
expect(() => sanitizeType(makeForeign())).toThrow(
`The Arrow type ${name} requires apache-arrow`,
);
},
);
it("should sanitize IntervalMonthDayNano to a generic Interval", () => {
// The subclass shares `Type.Interval`, which every supported Arrow
// release has; only the unit is new.
const result = sanitizeType(new arrow21.IntervalMonthDayNano());
expect(result).toBeInstanceOf(arrow.Interval);
expect((result as arrow.Interval).unit).toBe(
arrow21.IntervalUnit.MONTH_DAY_NANO,
);
});
});
it("should rebuild a schema from a second copy of the same Arrow version", function () {
// Arrow 21 makes `instanceof Schema` succeed across copies of the library
// but not `instanceof <DataType>`; the schema must still be rebuilt.
const foreign = new arrow21.Schema([
new arrow21.Field("x", new arrow21.Float32(), true),
]);
// Only Arrow 21+ overrides `Symbol.hasInstance` this way.
const crossCopyInstanceof = Object.getOwnPropertySymbols(
arrow.Schema,
).includes(Symbol.hasInstance);
expect(foreign instanceof arrow.Schema).toBe(crossCopyInstanceof);
const result = sanitizeSchema(foreign);
expect(result).not.toBe(foreign);
const type = result.fields[0].type;
expect(type).toBeInstanceOf(arrow.DataType);
expect(arrow.DataType.isFloat(type)).toBe(true);
expect((type as arrow.Float).precision).toBe(arrow.Precision.SINGLE);
expect(() => arrow.makeBuilder({ type })).not.toThrow();
});
});
describe("sanitizeField function", function () {
nodejs/__test__/table.test.ts
+861 -18
View File
@@ -2,22 +2,21 @@
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import * as fs from "fs";
import { getEventListeners } from "node:events";
import { pathToFileURL } from "node:url";
import * as path from "path";
import * as tmp from "tmp";
import * as arrow15 from "apache-arrow-15";
import * as arrow16 from "apache-arrow-16";
import * as arrow17 from "apache-arrow-17";
import * as arrow18 from "apache-arrow-18";
import {
AutoQuery,
type BlobInput,
Connection,
MatchQuery,
PhraseQuery,
Query,
Table,
VectorQuery,
blob,
connect,
tokenize,
} from "../lancedb";
@@ -52,19 +51,22 @@ import {
Operator,
instanceOfFullTextQuery,
} from "../lancedb/query";
import { LocalTable } from "../lancedb/table";
import {
type ApacheArrow,
arrow18,
arrowVersions,
latestArrow,
} from "./arrow_versions";
describe.each([arrow15, arrow16, arrow17, arrow18])(
describe.each(arrowVersions)(
"Given a table",
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
(arrow: any) => {
let tmpDir: tmp.DirResult;
let table: Table;
const schema:
| import("apache-arrow-15").Schema
| import("apache-arrow-16").Schema
| import("apache-arrow-17").Schema
| import("apache-arrow-18").Schema = new arrow.Schema([
const schema: InstanceType<ApacheArrow["Schema"]> = new arrow.Schema([
new arrow.Field("id", new arrow.Float64(), true),
]);
@@ -89,6 +91,38 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
await expect(table.countRows()).resolves.toBe(3);
});
it("creates and adds Arrow tables with large string and binary columns", async () => {
const data = new arrow.Table({
text: arrow.vectorFromArray(["alpha", "beta"], new arrow.LargeUtf8()),
bytes: arrow.vectorFromArray(
[Buffer.from("one"), Buffer.from("two")],
new arrow.LargeBinary(),
),
});
const conn = await connect(tmpDir.name);
const largeTable = await conn.createTable("large_columns", data);
await largeTable.add(data);
const rows = await largeTable.query().toArray();
expect(rows.map((row) => row.text)).toEqual([
"alpha",
"beta",
"alpha",
"beta",
]);
expect(rows.map((row) => Buffer.from(row.bytes).toString())).toEqual([
"one",
"two",
"one",
"two",
]);
const fields = (await largeTable.schema()).fields;
expect(fields.map((field) => field.type.typeId)).toEqual([
arrow.Type.LargeUtf8,
arrow.Type.LargeBinary,
]);
});
it("should support a foreign Float64 vector schema end to end", async () => {
const conn = await connect(tmpDir.name);
const schema = new arrow.Schema([
@@ -281,7 +315,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
numIndices: 0,
numRows: 3,
// Full on-disk size of the two data files, footers and metadata included.
totalBytes: 684,
totalBytes: 550,
});
// Index files count toward totalBytes too (only deletion files and
@@ -289,7 +323,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
await table.createIndex("id", { config: Index.btree() });
const statsWithIndex = await table.stats();
expect(statsWithIndex.numIndices).toBe(1);
expect(statsWithIndex.totalBytes).toBeGreaterThan(684);
expect(statsWithIndex.totalBytes).toBeGreaterThan(550);
});
it("should overwrite data if asked", async () => {
@@ -414,6 +448,26 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
expect(await table.countRows("id == 10")).toBe(1);
});
it("should let me update float values to NaN and Infinity", async () => {
const db = await connect(tmpDir.name);
const floats = await db.createTable("floats", [
{ id: 1, price: 1.5 },
{ id: 2, price: 2.5 },
{ id: 3, price: 3.5 },
{ id: 4, price: 4.5 },
]);
await floats.update({ values: { price: NaN }, where: "id = 1" });
await floats.update({ values: { price: Infinity }, where: "id = 2" });
await floats.update({ values: { price: -Infinity }, where: "id = 3" });
const rows = await floats.query().toArray();
const price = (id: number) => rows.find((row) => row.id === id).price;
expect(price(1)).toBeNaN();
expect(price(2)).toBe(Infinity);
expect(price(3)).toBe(-Infinity);
expect(price(4)).toBe(4.5);
});
// https://github.com/lancedb/lancedb/issues/1293
test.each([new arrow.Float16(), new arrow.Float32(), new arrow.Float64()])(
"can create empty table with non default float type: %s",
@@ -686,6 +740,114 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
);
// https://github.com/lancedb/lancedb/issues/1963
// These types were added in Arrow 21, so they are built with a second copy of
// that release rather than the per-version matrix above, and reading them back
// needs the library's own Arrow to know them too.
const hostArrowSupportsViewTypes =
typeof (arrow as unknown as Record<string, unknown>).Utf8View === "function";
(hostArrowSupportsViewTypes ? it : it.skip)(
"creates and adds tables with LargeList, Utf8View and BinaryView columns",
async () => {
const tmpDir = tmp.dirSync({ unsafeCleanup: true });
const db = await connect(tmpDir.name);
const makeData = () =>
new latestArrow.Table({
id: latestArrow.vectorFromArray([1, 2], new latestArrow.Int32()),
tags: latestArrow.vectorFromArray(
[["a", "b"], ["c"]],
new latestArrow.LargeList(
new latestArrow.Field("item", new latestArrow.Utf8(), true),
),
),
text: latestArrow.vectorFromArray(
["short", "a string that is longer than the twelve inline bytes"],
new latestArrow.Utf8View(),
),
bytes: latestArrow.vectorFromArray(
[new Uint8Array([1, 2, 3]), new Uint8Array(20).fill(7)],
new latestArrow.BinaryView(),
),
});
const table = await db.createTable("views", makeData());
await table.add(makeData());
const schema = await table.schema();
// Take the enum from the Arrow 21 copy: the host Arrow may predate
// `Type.LargeList`, and this file still has to type-check there.
expect(schema.fields.map((f) => f.type.typeId)).toEqual([
latestArrow.Type.Int,
latestArrow.Type.LargeList,
// Lance stores the view types as their offset-based equivalents.
latestArrow.Type.Utf8,
latestArrow.Type.Binary,
]);
const rows = await table.query().toArrow();
expect(rows.numRows).toBe(4);
const row = rows.get(1)!.toJSON();
expect(row.id).toBe(2);
expect(row.tags.toJSON()).toEqual(["c"]);
expect(row.text).toBe(
"a string that is longer than the twelve inline bytes",
);
expect(Array.from(row.bytes)).toEqual(Array(20).fill(7));
// Arrow JS 21 serializes a *sliced* view column with a truncated views
// buffer; the library rebuilds view columns as Utf8/Binary so this works.
const sliced = makeData().slice(1, 2);
expect(sliced.getChild("text")!.data[0].offset).toBe(1);
const slicedTable = await db.createTable("views_sliced", sliced);
await slicedTable.add(sliced);
const slicedRows = await slicedTable.query().toArrow();
expect(slicedRows.numRows).toBe(2);
for (const i of [0, 1]) {
const r = slicedRows.get(i)!.toJSON();
expect(r.id).toBe(2);
expect(r.text).toBe(
"a string that is longer than the twelve inline bytes",
);
expect(Array.from(r.bytes)).toEqual(Array(20).fill(7));
}
// Regression: a field with an integer-like name. JavaScript enumerates
// such keys first, so a key-ordered rebuild would pair "1" with the text
// column's values. The batches are built explicitly to pin field order.
const numericSchema = new latestArrow.Schema([
new latestArrow.Field("text", new latestArrow.Utf8View(), true),
new latestArrow.Field("1", new latestArrow.Utf8View(), true),
new latestArrow.Field("n", new latestArrow.Int32(), true),
]);
const column = (
values: unknown[],
type: InstanceType<typeof latestArrow.DataType>,
) => latestArrow.vectorFromArray(values, type).data[0];
const numericBatch = new latestArrow.RecordBatch(
numericSchema,
latestArrow.makeData({
type: new latestArrow.Struct(numericSchema.fields),
length: 3,
nullCount: 0,
children: [
column(["t0", "t1", "t2"], new latestArrow.Utf8View()),
column(["one0", "one1", "one2"], new latestArrow.Utf8View()),
column([0, 1, 2], new latestArrow.Int32()),
],
}),
);
const numericData = new latestArrow.Table(numericSchema, [numericBatch]);
const numericTable = await db.createTable("views_numeric", numericData);
await numericTable.add(numericData.slice(1));
const numericRows = (await numericTable.query().toArrow()).toArray();
expect(numericRows.map((r) => r.toJSON())).toEqual([
{ text: "t0", "1": "one0", n: 0 },
{ text: "t1", "1": "one1", n: 1 },
{ text: "t2", "1": "one2", n: 2 },
{ text: "t1", "1": "one1", n: 1 },
{ text: "t2", "1": "one2", n: 2 },
]);
tmpDir.removeCallback();
},
);
it("should query documents with LangChain PDF metadata", async () => {
const tmpDir = tmp.dirSync({ unsafeCleanup: true });
try {
@@ -1050,6 +1212,12 @@ describe("When creating an index", () => {
.toArrow();
expect(rst.numRows).toBe(2);
expect(() => tbl.vectorSearch(queryVec).nprobes(0)).toThrow(
"Invalid input, nprobes must be greater than 0",
);
expect(() =>
tbl.query().nearestTo(queryVec).fullTextSearch("dog").nprobes(0),
).toThrow("Invalid input, nprobes must be greater than 0");
expect(() => tbl.query().nearestTo(queryVec).minimumNprobes(0)).toThrow(
"Invalid input, minimum_nprobes must be greater than 0",
);
@@ -2401,6 +2569,641 @@ describe("when dealing with versioning", () => {
});
});
describe("when dealing with blob columns", () => {
let tmpDir: tmp.DirResult;
beforeEach(() => {
tmpDir = tmp.dirSync({ unsafeCleanup: true });
});
afterEach(() => {
tmpDir.removeCallback();
});
it("rejects unsupported blob values in createTable and add", async () => {
const db = await connect(tmpDir.name);
const schema = new Schema([
new Field("id", new Int64(), true),
blob("image"),
]);
const row = { id: 1n, image: new ReadableStream() };
await expect(db.createTable("invalid", [row], { schema })).rejects.toThrow(
/field image at row 0/,
);
const table = await db.createEmptyTable("empty", schema);
await expect(table.add([row])).rejects.toThrow(/field image at row 0/);
await expect(table.countRows()).resolves.toBe(0);
});
it("requires an explicit opt-in for blob URIs outside registered bases", async () => {
const { table, uri } = await openExternalBlobTable();
await expect(table.add([{ id: 1n, payload: uri }])).rejects.toThrow(
"allow_external_blob_outside_bases",
);
await expect(table.countRows()).resolves.toBe(0);
await expect(
table.add([{ id: 2n, payload: uri }], {
allowExternalBlobOutsideBases: false,
}),
).rejects.toThrow("allow_external_blob_outside_bases");
await expect(table.countRows()).resolves.toBe(0);
});
it.each([
["URI string", (uri: string) => uri],
["URI struct", (uri: string) => ({ uri })],
["URL", (uri: string) => new URL(uri)],
["URL struct", (uri: string) => ({ uri: new URL(uri) })],
])(
"round-trips an external blob from a %s after reopening",
async (_label, blobValue) => {
const { db, table, uri, payload } = await openExternalBlobTable();
const result = await table.add([{ id: 1n, payload: blobValue(uri) }], {
allowExternalBlobOutsideBases: true,
});
expect(result.version).toBeGreaterThan(0);
await expect(table.countRows()).resolves.toBe(1);
table.close();
const reopened = await db.openTable("external_blobs");
const rows = await reopened.query().withRowId().toArray();
expect(rows).toHaveLength(1);
expect(rows[0].payload.blob_uri).toBe(uri);
const bytes = await reopened.fetchBlobs("payload", [
rows[0]._rowid as bigint,
]);
expect(bytes[0]).toEqual(payload);
},
);
it("applies the external blob opt-in to Arrow table input", async () => {
const { table, schema, uri, payload } = await openExternalBlobTable();
const input = makeArrowTable([{ id: 1n, payload: uri }], { schema });
const result = await table.add(input, {
allowExternalBlobOutsideBases: true,
});
expect(result.version).toBeGreaterThan(0);
const rows = await table.query().withRowId().toArray();
expect(rows[0].payload.blob_uri).toBe(uri);
const bytes = await table.fetchBlobs("payload", [rows[0]._rowid as bigint]);
expect(bytes[0]).toEqual(payload);
});
it("applies the external blob opt-in only to the current add", async () => {
const { table, uri } = await openExternalBlobTable();
await table.add([{ id: 1n, payload: uri }], {
allowExternalBlobOutsideBases: true,
});
await expect(table.countRows()).resolves.toBe(1);
await expect(table.add([{ id: 2n, payload: uri }])).rejects.toThrow(
"allow_external_blob_outside_bases",
);
await expect(table.countRows()).resolves.toBe(1);
});
it("supports progress and overwrite with the external blob opt-in", async () => {
const { table, uri, payload } = await openExternalBlobTable();
await table.add([{ id: 1n, payload: Buffer.from("old") }]);
let resolveFinal!: (progress: import("../lancedb").WriteProgress) => void;
let rejectFinal!: (error: Error) => void;
const finalProgress = new Promise<import("../lancedb").WriteProgress>(
(resolve, reject) => {
resolveFinal = resolve;
rejectFinal = reject;
},
);
const timeout = setTimeout(
() => rejectFinal(new Error("timed out waiting for final progress")),
5_000,
);
try {
const result = await table.add([{ id: 2n, payload: uri }], {
mode: "overwrite",
allowExternalBlobOutsideBases: true,
progress: (progress) => {
if (progress.done) {
resolveFinal(progress);
}
},
});
expect(result.version).toBeGreaterThan(0);
const progress = await finalProgress;
expect(progress.outputRows).toBe(1);
expect(progress.totalRows).toBe(1);
} finally {
clearTimeout(timeout);
}
const rows = await table.query().withRowId().toArray();
expect(rows).toHaveLength(1);
expect(rows[0].id).toBe(2n);
const bytes = await table.fetchBlobs("payload", [rows[0]._rowid as bigint]);
expect(bytes[0]).toEqual(payload);
});
it("accepts ArrayBuffer, Blob, and File in createTable and add", async () => {
const db = await connect(tmpDir.name);
const schema = new Schema([
new Field("id", new Int64(), true),
blob("image"),
]);
const rows: { id: bigint; image: BlobInput }[] = [
{ id: 1n, image: new TextEncoder().encode("array-buffer").buffer },
{ id: 2n, image: new Blob(["blob"]) },
{ id: 3n, image: { data: new File(["file"], "f.txt") } },
];
const table = await db.createTable("widened", rows.slice(0, 2), {
schema,
});
await table.add(rows.slice(2));
const results = await table.query().select(["id"]).withRowId().toArray();
results.sort((a, b) => Number(a.id - b.id));
const bytes = await table.fetchBlobs(
"image",
results.map((row) => row._rowid as bigint),
);
expect(bytes.map((b) => b?.toString())).toEqual([
"array-buffer",
"blob",
"file",
]);
});
it("accepts Blob values in mergeInsert", async () => {
const db = await connect(tmpDir.name);
const schema = new Schema([
new Field("id", new Int64(), true),
blob("image"),
]);
const table = await db.createTable(
"merge_blobs",
[{ id: 1n, image: Buffer.from("old") }],
{ schema },
);
await table
.mergeInsert("id")
.whenMatchedUpdateAll()
.whenNotMatchedInsertAll()
.execute([
{ id: 1n, image: new Blob(["new"]) },
{ id: 2n, image: new Uint8Array([104, 105]).buffer },
]);
const results = await table.query().select(["id"]).withRowId().toArray();
results.sort((a, b) => Number(a.id - b.id));
const bytes = await table.fetchBlobs(
"image",
results.map((row) => row._rowid as bigint),
);
expect(bytes.map((b) => b?.toString())).toEqual(["new", "hi"]);
});
it("discovers blob columns", async () => {
const { table } = await openBlobTable();
expect(await table.blobColumns()).toEqual(["image"]);
});
it("preserves order, duplicates, and nulls", async () => {
const { table, rowIds } = await openBlobTable();
const [alphaId, betaId, nullId] = rowIds;
const bytes = await table.fetchBlobs("image", [
betaId,
alphaId,
betaId,
nullId,
]);
expect(bytes.map((b) => (b == null ? null : b.toString()))).toEqual([
"beta",
"alpha",
"beta",
null,
]);
const files = await table.fetchBlobFiles("image", [
betaId,
nullId,
alphaId,
]);
expect(files.map((f) => f == null)).toEqual([false, true, false]);
});
it("reads full blob contents", async () => {
const { table, rowIds, alpha, beta } = await openBlobTable();
const bytes = await table.fetchBlobs("image", rowIds);
expect(bytes[0]!.equals(alpha)).toBe(true);
expect(bytes[1]!.equals(beta)).toBe(true);
const files = await table.fetchBlobFiles("image", rowIds);
expect(files[0]!.size()).toBe(BigInt(alpha.length));
expect(Buffer.from(await files[0]!.read()).toString()).toBe("alpha");
expect(Buffer.from(await files[1]!.read()).toString()).toBe("beta");
});
it("adds a LargeBinary Arrow column to a blob table", async () => {
const db = await connect(tmpDir.name);
const schema = new Schema([new Field("id", new Int64()), blob("image")]);
const table = await db.createTable(
"blobs",
[{ id: 1n, image: Buffer.from("alpha") }],
{ schema },
);
const payload = Buffer.from("beta");
const data = new arrow18.Table({
id: arrow18.vectorFromArray([2n], new arrow18.Int64()),
image: arrow18.vectorFromArray([payload], new arrow18.LargeBinary()),
});
await table.add(data);
const rows = await table.query().withRowId().toArray();
const rowId = rows.find((row) => row.id === 2n)!._rowid as bigint;
const [actual] = await table.fetchBlobs("image", [rowId]);
expect(actual).toEqual(payload);
});
it("reads a half-open range", async () => {
const { table, rowIds } = await openBlobTable();
const files = await table.fetchBlobFiles("image", rowIds);
expect(Buffer.from(await files[0]!.readRange(0n, 2n)).toString()).toBe(
"al",
);
});
it("readRange does not move the cursor", async () => {
const { table, rowIds, alpha } = await openBlobTable();
const [handle] = await table.fetchBlobFiles("image", rowIds);
expect((await handle!.readRange(1n, 3n)).toString()).toBe("lp");
expect(await handle!.read()).toEqual(alpha);
expect(await handle!.read()).toEqual(Buffer.alloc(0));
});
it("fails when readRange end is past the blob size", async () => {
const { table, rowIds, alpha } = await openBlobTable();
const files = await table.fetchBlobFiles("image", rowIds);
await expect(
files[0]!.readRange(0n, BigInt(alpha.length + 1)),
).rejects.toThrow(/exceeds blob size/);
});
it("reads at most maxBytes and advances the cursor", async () => {
const { table, rowIds } = await openBlobTable();
const [handle] = await table.fetchBlobFiles("image", rowIds);
expect((await handle!.read(2n)).toString()).toBe("al");
expect(await handle!.tell()).toBe(2n);
expect((await handle!.read(2n)).toString()).toBe("ph");
expect((await handle!.read(10n)).toString()).toBe("a");
expect(await handle!.tell()).toBe(5n);
expect(await handle!.read(1n)).toEqual(Buffer.alloc(0));
});
it("seeks the cursor", async () => {
const { table, rowIds, alpha } = await openBlobTable();
const [handle] = await table.fetchBlobFiles("image", rowIds);
await handle!.seek(3n);
expect(await handle!.tell()).toBe(3n);
expect((await handle!.read()).toString()).toBe("ha");
await handle!.seek(0n);
expect(await handle!.read()).toEqual(alpha);
});
it("rejects a negative seek position", async () => {
const { table, rowIds } = await openBlobTable();
const [handle] = await table.fetchBlobFiles("image", rowIds);
await expect(handle!.seek(-1n)).rejects.toThrow(/cannot be negative/);
});
it("readRanges returns one buffer per range in order", async () => {
const { table, rowIds } = await openBlobTable();
const [handle] = await table.fetchBlobFiles("image", rowIds);
const buffers = await handle!.readRanges([
{ start: 3n, end: 5n },
{ start: 0n, end: 2n },
{ start: 3n, end: 5n },
]);
expect(buffers.map((b) => b.toString())).toEqual(["ha", "al", "ha"]);
expect(await handle!.tell()).toBe(0n);
});
it("fails when a readRanges end is past the blob size", async () => {
const { table, rowIds, alpha } = await openBlobTable();
const [handle] = await table.fetchBlobFiles("image", rowIds);
await expect(
handle!.readRanges([
{ start: 0n, end: 1n },
{ start: 0n, end: BigInt(alpha.length + 1) },
]),
).rejects.toThrow(/exceeds blob size/);
});
it("fails reads after close and closes once", async () => {
const { table, rowIds } = await openBlobTable();
const [handle] = await table.fetchBlobFiles("image", rowIds);
expect(handle!.isClosed()).toBe(false);
await handle!.close();
expect(handle!.isClosed()).toBe(true);
await expect(handle!.close()).resolves.toBeUndefined();
await expect(handle!.read()).rejects.toThrow(/already closed/);
await expect(handle!.read(1n)).rejects.toThrow(/already closed/);
await expect(handle!.readRange(0n, 1n)).rejects.toThrow(/already closed/);
await expect(handle!.readRanges([{ start: 0n, end: 1n }])).rejects.toThrow(
/already closed/,
);
await expect(handle!.seek(0n)).rejects.toThrow(/already closed/);
await expect(handle!.tell()).rejects.toThrow(/already closed/);
expect(handle!.size()).toBe(5n);
});
it("rejects reads with an already aborted signal", async () => {
const { table, rowIds, alpha } = await openBlobTable();
const [handle] = await table.fetchBlobFiles("image", rowIds);
const signal = AbortSignal.abort();
const abortError = { name: "AbortError" };
await expect(handle!.read(undefined, { signal })).rejects.toMatchObject(
abortError,
);
await expect(handle!.read(2n, { signal })).rejects.toMatchObject(
abortError,
);
await expect(handle!.readRange(0n, 1n, { signal })).rejects.toMatchObject(
abortError,
);
await expect(
handle!.readRanges([{ start: 0n, end: 1n }], { signal }),
).rejects.toMatchObject(abortError);
await expect(
table.fetchBlobs("image", rowIds, { signal }),
).rejects.toMatchObject(abortError);
await expect(
table.fetchBlobFiles("image", rowIds, { signal }),
).rejects.toMatchObject(abortError);
expect(await handle!.read()).toEqual(alpha);
});
it("rejects with the abort reason", async () => {
const { table, rowIds } = await openBlobTable();
const [handle] = await table.fetchBlobFiles("image", rowIds);
const reason = new Error("caller gave up");
await expect(
handle!.readRange(0n, 1n, { signal: AbortSignal.abort(reason) }),
).rejects.toBe(reason);
});
it("reads with a signal and removes its abort listener", async () => {
const { table, rowIds, alpha, beta } = await openBlobTable();
const controller = new AbortController();
const { signal } = controller;
const [handle] = await table.fetchBlobFiles("image", rowIds, { signal });
expect((await handle!.readRange(0n, 2n, { signal })).toString()).toBe("al");
expect(await table.fetchBlobs("image", rowIds, { signal })).toEqual([
alpha,
beta,
null,
]);
expect(getEventListeners(signal, "abort")).toHaveLength(0);
controller.abort();
expect(await handle!.read()).toEqual(alpha);
});
it("cancels an in-flight read", async () => {
const db = await connect(tmpDir.name);
const schema = new Schema([
new Field("id", new Int64(), true),
blob("payload", { inlineSizeThreshold: 0, dedicatedSizeThreshold: 1 }),
]);
const payload = Buffer.alloc(32 * 1024 * 1024, 7);
const table = await db.createTable("large_blobs", [{ id: 1n, payload }], {
schema,
});
const [row] = await table.query().withRowId().toArray();
const [handle] = await table.fetchBlobFiles("payload", [
row._rowid as bigint,
]);
const controller = new AbortController();
const pending = handle!.readRange(0n, handle!.size(), {
signal: controller.signal,
});
controller.abort();
await expect(pending).rejects.toMatchObject({ name: "AbortError" });
expect(await handle!.readRange(0n, 4n)).toEqual(payload.subarray(0, 4));
});
it("rejects fetchBlobs on a non-blob column", async () => {
const { table, rowIds } = await openBlobTable();
await expect(table.fetchBlobs("id", rowIds)).rejects.toThrow(/blob/i);
});
it("discovers and fetches nested blob columns", async () => {
const db = await connect(tmpDir.name);
const schema = new Schema([
new Field("id", new Int64(), true),
new Field("info", new Struct([blob("image")]), true),
]);
const payload = Buffer.from("nested");
const table = await db.createTable(
"nested_blobs",
[{ id: 1n, info: { image: payload } }],
{ schema },
);
expect(await table.blobColumns()).toEqual(["info.image"]);
const rows = await table.query().withRowId().toArray();
const bytes = await table.fetchBlobs("info.image", [
rows[0]._rowid as bigint,
]);
expect(bytes[0]!.equals(payload)).toBe(true);
});
it("creates and adds list blob columns", async () => {
const db = await connect(tmpDir.name);
const schema = new Schema([
new Field("id", new Int64(), true),
new Field("images", new List(blob("image")), true),
]);
const alpha = Buffer.from("alpha");
const beta = Buffer.from("beta");
const gamma = Buffer.from("gamma");
const table = await db.createTable(
"list_blobs",
[{ id: 1n, images: [alpha, beta] }],
{ schema },
);
await table.add([
{ id: 2n, images: null },
{ id: 3n, images: [gamma, null] },
{ id: 4n, images: [] },
]);
expect(await table.blobColumns()).toEqual(["images.image"]);
const rows = await table.query().toArray();
const byId = new Map(rows.map((row) => [Number(row.id), row]));
expect(descriptorSizes(byId.get(1)!.images)).toEqual([
alpha.length,
beta.length,
]);
expect(byId.get(2)!.images).toBeNull();
expect(descriptorSizes(byId.get(3)!.images)).toEqual([gamma.length, null]);
expect(Array.from(byId.get(4)!.images as Iterable<unknown>)).toHaveLength(
0,
);
});
it("creates and adds list struct blob columns", async () => {
const db = await connect(tmpDir.name);
const schema = new Schema([
new Field("id", new Int64(), true),
new Field(
"items",
new List(
new Field(
"item",
new Struct([new Field("name", new Utf8(), true), blob("image")]),
true,
),
),
true,
),
]);
const alpha = Buffer.from("nested-alpha");
const beta = Buffer.from("nested-beta");
const table = await db.createTable(
"list_struct_blobs",
[{ id: 1n, items: [{ name: "one", image: alpha }] }],
{ schema },
);
await table.add([
{
id: 2n,
items: [
{ name: "two", image: beta },
{ name: "three", image: null },
],
},
]);
const rows = await table.query().toArray();
const byId = new Map(rows.map((row) => [Number(row.id), row]));
expect(
descriptorSizes(
Array.from(byId.get(1)!.items as Iterable<{ image: unknown }>).map(
(item) => item.image,
),
),
).toEqual([alpha.length]);
expect(
descriptorSizes(
Array.from(byId.get(2)!.items as Iterable<{ image: unknown }>).map(
(item) => item.image,
),
),
).toEqual([beta.length, null]);
});
it("rejects blob fields inside a fixed-size list", async () => {
const db = await connect(tmpDir.name);
const schema = new Schema([
new Field("id", new Int64(), true),
new Field("frames", new FixedSizeList(2, blob("frame")), true),
]);
await expect(
db.createTable(
"fsl_blobs",
[{ id: 1n, frames: [Buffer.from("a"), Buffer.from("b")] }],
{ schema },
),
).rejects.toThrow(
"Blob fields inside FixedSizeList are not supported. Use List instead.",
);
});
it("rejects blob fields inside a nested fixed-size list", async () => {
const db = await connect(tmpDir.name);
const schema = new Schema([
new Field("id", new Int64(), true),
new Field(
"clip",
new Struct([
new Field("frames", new FixedSizeList(2, blob("frame")), true),
]),
true,
),
]);
await expect(
db.createTable(
"nested_fsl_blobs",
[
{
id: 1n,
clip: { frames: [Buffer.from("a"), Buffer.from("b")] },
},
],
{ schema },
),
).rejects.toThrow(
"Blob fields inside FixedSizeList are not supported. Use List instead.",
);
});
it("rejects an Arrow table with blob fields inside a fixed-size list", async () => {
const db = await connect(tmpDir.name);
const schema = new Schema([
new Field("id", new Int64(), true),
new Field("frames", new FixedSizeList(2, blob("frame")), true),
]);
await expect(
db.createTable("fsl_blobs_ipc", new ArrowTable(schema)),
).rejects.toThrow(
"Blob fields inside FixedSizeList are not supported. Use List instead.",
);
});
function descriptorSizes(values: unknown): (number | null)[] {
return Array.from(
values as Iterable<{ size?: bigint | number } | null>,
).map((value) => (value == null ? null : Number(value.size)));
}
async function openBlobTable() {
const db = await connect(tmpDir.name);
const schema = new Schema([
new Field("id", new Int64(), true),
blob("image"),
]);
const alpha = Buffer.from("alpha");
const beta = Buffer.from("beta");
const table = await db.createTable(
"blobs",
[
{ id: 1n, image: alpha },
{ id: 2n, image: beta },
{ id: 3n, image: null },
],
{ schema },
);
const rows = await table.query().withRowId().toArray();
const rowIdById = new Map(
rows.map((r) => [Number(r.id), r._rowid as bigint]),
);
const rowIds = [1, 2, 3].map((id) => rowIdById.get(id)!);
return { table, rowIds, alpha, beta };
}
async function openExternalBlobTable() {
const db = await connect(path.join(tmpDir.name, "db"));
const payload = Buffer.from("external-payload");
const payloadPath = path.join(tmpDir.name, "payload.bin");
fs.writeFileSync(payloadPath, payload);
const uri = pathToFileURL(payloadPath).href;
const schema = new Schema([
new Field("id", new Int64(), true),
blob("payload"),
]);
const table = await db.createEmptyTable("external_blobs", schema);
return { db, table, schema, uri, payload };
}
});
describe("when dealing with tags", () => {
let tmpDir: tmp.DirResult;
beforeEach(() => {
@@ -2494,6 +3297,15 @@ describe("when dealing with tags", () => {
});
});
/** Returns a Date strictly later than every instant observed before the call. */
async function nextMillisecond(): Promise<Date> {
const start = Date.now();
while (Date.now() <= start) {
await new Promise((resolve) => setTimeout(resolve, 1));
}
return new Date();
}
describe("when optimizing a dataset", () => {
let tmpDir: tmp.DirResult;
let table: Table;
@@ -2516,9 +3328,14 @@ describe("when optimizing a dataset", () => {
});
it("cleanups old versions", async () => {
const stats = await table.optimize({ cleanupOlderThan: new Date() });
// Lance stores version timestamps with nanosecond precision while a JS
// Date only has millisecond precision. A cutoff captured in the same
// millisecond as the last commit would truncate to *before* that commit
// and leave it in place, so wait for the clock to tick over first.
const cutoff = await nextMillisecond();
const stats = await table.optimize({ cleanupOlderThan: cutoff });
expect(stats.prune.bytesRemoved).toBeGreaterThan(0);
expect(stats.prune.oldVersionsRemoved).toBe(3);
expect(stats.prune.oldVersionsRemoved).toBe(2);
});
it("delete unverified", async () => {
@@ -2539,7 +3356,25 @@ describe("when optimizing a dataset", () => {
});
});
describe.each([arrow15, arrow16, arrow17, arrow18])(
it("passes cleanupOlderThan to the native binding as an absolute timestamp", async () => {
const optimize = jest.fn().mockResolvedValue({
compaction: {
filesAdded: 0,
filesRemoved: 0,
fragmentsAdded: 0,
fragmentsRemoved: 0,
},
prune: { bytesRemoved: 0, oldVersionsRemoved: 0 },
});
const table = new LocalTable({ optimize } as never);
const cutoff = new Date("2020-01-02T03:04:05.678Z");
await table.optimize({ cleanupOlderThan: cutoff, deleteUnverified: true });
expect(optimize).toHaveBeenCalledWith(cutoff.getTime(), true);
});
describe.each(arrowVersions)(
"when optimizing a dataset",
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
(arrow: any) => {
@@ -3252,7 +4087,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
const db = await connect(tmpDir.name);
const data = [
{ text: "fa", vector: [0.1, 0.2, 0.3] },
{ text: "fo", vector: [0.4, 0.5, 0.6] },
{ text: "fo", vector: [0.4, 0.5, 0.6] }, // spellchecker:disable-line
{ text: "fob", vector: [0.4, 0.5, 0.6] },
{ text: "focus", vector: [0.4, 0.5, 0.6] },
{ text: "foo", vector: [0.4, 0.5, 0.6] },
@@ -3277,7 +4112,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
const resultSet = new Set(fuzzyResults.map((r) => r.text));
expect(resultSet.has("foo")).toBe(true);
expect(resultSet.has("fob")).toBe(true);
expect(resultSet.has("fo")).toBe(true);
expect(resultSet.has("fo")).toBe(true); // spellchecker:disable-line
expect(resultSet.has("food")).toBe(true);
const prefixResults = await table
@@ -4054,6 +4889,14 @@ describe("computed columns", () => {
expect(rows.map((r) => r.doubled).sort()).toEqual([2, 4]);
});
it("records Function errors only on remote tables", async () => {
const db = await connect(tmpDir.name);
const table = await db.createTable("errors_local", [{ x: 1 }]);
await expect(table.functionErrors()).rejects.toThrow(
"LanceDB Cloud and Enterprise",
);
});
it("returns a job handle from refreshColumnAsync", async () => {
const db = await connect(tmpDir.name);
const table = await db.createTable("computed_job", [{ x: 1 }, { x: 2 }]);
nodejs/__test__/util.test.ts
+3
View File
@@ -14,6 +14,9 @@ test.each([
[new ArrayBuffer(8), "X'0000000000000000'"],
[Buffer.from("hello"), "X'68656c6c6f'"],
["Hello 'world'", "'Hello ''world'''"],
[NaN, "CAST('NaN' AS DOUBLE)"],
[Infinity, "CAST('Infinity' AS DOUBLE)"],
[-Infinity, "CAST('-Infinity' AS DOUBLE)"],
])("toSQL(%p) === %p", (value, expected) => {
expect(toSQL(value)).toBe(expected);
});
nodejs/jest.config.js
+9
View File
@@ -1,3 +1,12 @@
const path = require("node:path");
// Set the real process environment before Jest creates its sandbox and workers.
// Assigning process.env inside a test does not reach Rust's std::env.
process.env.LANCEDB_OAUTH_BROWSER =
process.platform === "win32"
? path.join(__dirname, "__test__/fixtures/oauth_browser.cmd")
: "/usr/bin/true";
/** @type {import('ts-jest').JestConfigWithTsJest} */
module.exports = {
preset: "ts-jest",
nodejs/lancedb/arrow.ts
+383 -17
View File
@@ -40,6 +40,12 @@ import {
} from "apache-arrow";
import { Buffers } from "apache-arrow/data";
import { typedArrayToArrowType } from "./arrow_type";
import {
blobToRead,
coerceBlobValue,
isBlobField,
withBlobBytes,
} from "./blob";
import { type EmbeddingFunction } from "./embedding/embedding_function";
import {
EmbeddingFunctionConfig,
@@ -65,11 +71,44 @@ export type SchemaLike =
export type FieldLike =
| Field
| {
type: string;
type: string | DataTypeLike;
name: string;
nullable: boolean;
metadata?: Map<string, string>;
};
/**
* A `DataType` from any copy or version of apache-arrow.
*
* Arrow 21 brands its classes with `unique symbol` properties, so a type
* object from a second copy of the library no longer satisfies the `DataType`
* type of this one even though it is structurally identical. Inputs that only
* need to be sanitized accept this looser shape instead.
*/
export type DataTypeLike = DataType | { readonly typeId: number };
/**
* Create an Arrow field backed by LanceDB's JSON extension type.
*
* @param name - The field name.
* @param nullable - Whether the field accepts null values.
* @example
* ```ts
* import { connect, makeJsonField } from "@lancedb/lancedb";
* import { Schema } from "apache-arrow";
*
* const schema = new Schema([makeJsonField("metadata")]);
* const db = await connect("/path/to/database");
* await db.createTable("items", [{ metadata: '{"source":"api"}' }], { schema });
* ```
*/
export function makeJsonField(name: string, nullable = true): Field {
return new Field(
name,
new Utf8(),
nullable,
new Map([["ARROW:extension:name", "arrow.json"]]),
);
}
export type DataLike =
| import("apache-arrow").Data
@@ -415,6 +454,24 @@ export function makeArrowTable(
);
}
if (schema !== undefined) {
// Validate blob values up front and give every one the full
// `{ data, uri }` shape, so inference sees the same struct whether a row
// passed bytes, a URI, or a partial struct.
data = mapBlobInputs(data, schema, (value, field, row) => {
if (value === undefined) {
return value;
}
try {
return coerceBlobValue(value);
} catch (e) {
throw new Error(
`Invalid value for blob field ${field} at row ${row}: ${(e as Error).message}`,
);
}
});
}
let schemaMetadata = schema?.metadata || new Map<string, string>();
if (metadata !== undefined) {
schemaMetadata = new Map([...schemaMetadata, ...metadata]);
@@ -430,12 +487,14 @@ export function makeArrowTable(
throw new Error("A schema must be provided if data is empty");
} else {
schema = new Schema(schema.fields, schemaMetadata);
validateBlobSchema(schema);
return new ArrowTable(schema);
}
}
let inferredSchema = inferSchema(data, schema, opt);
inferredSchema = new Schema(inferredSchema.fields, schemaMetadata);
validateBlobSchema(inferredSchema);
const finalColumns: Record<string, Vector> = {};
for (const field of inferredSchema.fields) {
@@ -445,6 +504,162 @@ export function makeArrowTable(
return new ArrowTable(inferredSchema, finalColumns);
}
function validateBlobSchema(schema: Schema): void {
for (const field of schema.fields) {
validateBlobField(field);
}
}
function validateBlobField(field: Field): void {
if (
isFixedSizeList(field.type) &&
containsBlobField(field.type.children[0])
) {
throw new Error(
"Blob fields inside FixedSizeList are not supported. Use List instead.",
);
}
for (const child of field.type.children ?? []) {
validateBlobField(child);
}
}
function containsBlobField(field: Field): boolean {
if (isBlobField(field)) {
return true;
}
return (field.type.children ?? []).some((child: Field) =>
containsBlobField(child),
);
}
type BlobInputVisitor = (value: unknown, field: string, row: number) => unknown;
/**
* Calls `visit` on every value that lands in a blob column of `schema`,
* including blobs nested in structs and lists, and replaces it with the
* result. Records and arrays are copied only where a value changed, so the
* caller's data is never mutated.
*/
function mapBlobInputs(
data: Array<Record<string, unknown>>,
schema: Schema,
visit: BlobInputVisitor,
): Array<Record<string, unknown>> {
if (!schema.fields.some(containsBlobField)) {
return data;
}
return data.map((record, row) =>
isObject(record)
? mapBlobFields(record, schema.fields, "", row, visit)
: record,
);
}
function mapBlobFields(
record: Record<string, unknown>,
fields: Field[],
prefix: string,
row: number,
visit: BlobInputVisitor,
): Record<string, unknown> {
let out: Record<string, unknown> | undefined;
for (const field of fields) {
if (!containsBlobField(field) || !Object.hasOwn(record, field.name)) {
continue;
}
const value = record[field.name];
const mapped = mapBlobField(
value,
field,
`${prefix}${field.name}`,
row,
visit,
);
if (mapped !== value) {
out ??= { ...record };
out[field.name] = mapped;
}
}
return out ?? record;
}
function mapBlobField(
value: unknown,
field: Field,
label: string,
row: number,
visit: BlobInputVisitor,
): unknown {
if (isBlobField(field)) {
return visit(value, label, row);
}
if (field.type instanceof Struct && isObject(value)) {
return mapBlobFields(value, field.type.children, `${label}.`, row, visit);
}
if (isList(field.type) && Array.isArray(value)) {
const child = field.type.children[0];
let out: unknown[] | undefined;
for (const [index, element] of value.entries()) {
const mapped = mapBlobField(
element,
child,
`${label}[${index}]`,
row,
visit,
);
if (mapped !== element) {
out ??= [...value];
out[index] = mapped;
}
}
return out ?? value;
}
return value;
}
/**
* Reads `Blob` / `File` values in the blob columns of `schema` into bytes so
* the synchronous conversion in {@link makeArrowTable} can accept them.
* Returns `data` itself when there is nothing to read.
*/
export async function resolveBlobInputs(
data: Array<Record<string, unknown>>,
schema?: SchemaLike,
): Promise<Array<Record<string, unknown>>> {
if (schema === undefined || schema === null) {
return data;
}
const sanitized = sanitizeSchema(schema);
// Keyed by the Blob itself, so rows sharing one Blob (bare or as
// `{ data }`) read it once and share the bytes.
const reads = new Map<Blob, Promise<Uint8Array>>();
mapBlobInputs(data, sanitized, (value) => {
const source = blobToRead(value);
if (source !== undefined && !reads.has(source)) {
reads.set(
source,
source.arrayBuffer().then((buffer) => new Uint8Array(buffer)),
);
}
return value;
});
if (reads.size === 0) {
return data;
}
const bytes = new Map(
await Promise.all(
[...reads].map(async ([source, read]) => [source, await read] as const),
),
);
return mapBlobInputs(data, sanitized, (value) => {
const source = blobToRead(value);
return source === undefined
? value
: withBlobBytes(value, bytes.get(source) as Uint8Array);
});
}
function isObject(value: unknown): value is Record<string, unknown> {
return (
typeof value === "object" &&
@@ -480,6 +695,32 @@ function transposeData(
path: string[] = [],
): Vector {
const valuesPath = [...path, field.name];
if (isBlobField(field) && field.type instanceof Struct) {
const blobRows = data.map((datum) =>
coerceBlobValue(valueAtPath(datum, valuesPath)),
);
const childVectors = field.type.children.map((child) => {
const values = blobRows.map((row) =>
row == null ? null : (row[child.name as "data" | "uri"] ?? null),
);
return makeVector(values, child.type, undefined, child.nullable);
});
const nullCount = blobRows.filter((row) => row === null).length;
const structData = makeData({
type: field.type,
length: blobRows.length,
nullCount,
nullBitmap:
nullCount > 0
? arrowUtil.packBools(blobRows.map((row) => row !== null))
: undefined,
children: childVectors.map((v) => v.data[0]),
});
return arrowMakeVector(structData);
}
if (isList(field.type) && containsBlobField(field.type.children[0])) {
return transposeListData(data, field, valuesPath);
}
const values = data.map((datum) => valueAtPath(datum, valuesPath));
if (field.type instanceof Struct) {
const childFields = field.type.children;
@@ -495,7 +736,7 @@ function transposeData(
nullCount > 0
? arrowUtil.packBools(values.map((value) => value !== null))
: undefined,
children: childVectors as unknown as ArrowData<DataType>[],
children: childVectors.map((v) => v.data[0]),
});
return arrowMakeVector(structData);
} else {
@@ -503,6 +744,48 @@ function transposeData(
}
}
function transposeListData(
data: Record<string, unknown>[],
field: Field,
valuesPath: string[],
): Vector {
const listType = field.type as List;
const childField = listType.children[0];
const lists = data.map((datum) => valueAtPath(datum, valuesPath));
const flattened: Record<string, unknown>[] = [];
const validity: boolean[] = [];
const offsets: number[] = [0];
for (const list of lists) {
if (list == null) {
validity.push(false);
offsets.push(flattened.length);
continue;
}
if (!Array.isArray(list)) {
throw new Error(`expected an array for list field '${field.name}'`);
}
validity.push(true);
for (const element of list) {
flattened.push({ [childField.name]: element });
}
offsets.push(flattened.length);
}
const childVector = transposeData(flattened, childField, []);
const nullCount = validity.filter((valid) => !valid).length;
return arrowMakeVector(
makeData({
type: listType,
length: lists.length,
nullCount,
nullBitmap: nullCount > 0 ? arrowUtil.packBools(validity) : undefined,
valueOffsets: Int32Array.from(offsets),
child: childVector.data[0],
}),
);
}
/**
* Create an empty Arrow table with the provided schema
*/
@@ -600,7 +883,7 @@ function makeVector(
}
if (values.length === 0) {
throw Error(
"makeVector requires at least one value or the type must be specfied",
"makeVector requires at least one value or the type must be specified",
);
}
const sampleValue = values.find((val) => val !== null && val !== undefined);
@@ -858,7 +1141,7 @@ async function applyEmbeddings<T>(
* customized by the `embeddingDataType` property of the embedding function.
*
* If a schema is provided in `makeTableOptions` then it should include the
* embedding columns. If no schema is provded then embedding columns will
* embedding columns. If no schema is provided then embedding columns will
* be placed at the end of the table, after all of the input columns.
*/
export async function convertToTable(
@@ -880,6 +1163,11 @@ export async function convertToTable(
);
}
processedData = await resolveBlobInputs(
processedData,
makeTableOptions?.schema,
);
const table = makeArrowTable(processedData, makeTableOptions);
return await applyEmbeddings(table, embeddings, makeTableOptions?.schema);
}
@@ -935,6 +1223,71 @@ export async function fromRecordsToStreamBuffer(
return Buffer.from(await writer.toUint8Array());
}
// `Type.Utf8View` / `Type.BinaryView` as numbers: the enum members only exist
// in Arrow 21+, and this module compiles against every supported release.
const UTF8_VIEW_TYPE_ID = 24;
const BINARY_VIEW_TYPE_ID = 23;
/**
* Copy a Utf8View / BinaryView `Data` into a single `Data` of `type`.
*/
function materializeViewData(child: ArrowData, type: DataType): ArrowData {
const builder = makeBuilder({ type, nullValues: [null] });
for (const value of new Vector([child])) {
builder.append(value);
}
return builder.finish().flush();
}
/**
* Rebuild any top-level Utf8View / BinaryView column as Utf8 / Binary.
*
* Lance stores the view types as their offset-based equivalents anyway, so
* nothing is lost. Doing it here also sidesteps an Arrow JS 21 bug: its IPC
* writer emits a truncated views buffer for a *sliced* view array, which the
* Rust reader rejects with "Need at least N bytes in buffers[0]".
*
* The record batches are rebuilt positionally rather than through a
* `Record<string, Vector>`: JavaScript enumerates integer-like keys first, so
* a field named e.g. `"1"` would otherwise be paired with the wrong column.
*
* Tables without view columns are returned as-is.
*/
function materializeViewColumns(table: ArrowTable): ArrowTable {
const replacements = new Map<number, DataType>();
table.schema.fields.forEach((field, i) => {
if (field.type.typeId === UTF8_VIEW_TYPE_ID) {
replacements.set(i, new Utf8());
} else if (field.type.typeId === BINARY_VIEW_TYPE_ID) {
replacements.set(i, new Binary());
}
});
if (replacements.size === 0) {
return table;
}
const fields = table.schema.fields.map((field, i) => {
const type = replacements.get(i);
return type === undefined
? field
: new Field(field.name, type, field.nullable, field.metadata);
});
const schema = new Schema(fields, table.schema.metadata);
const batches = table.batches.map((batch) => {
const children = batch.data.children.map((child, i) => {
const type = replacements.get(i);
return type === undefined ? child : materializeViewData(child, type);
});
const data = makeData({
type: new Struct(fields),
length: batch.numRows,
nullCount: 0,
children,
});
return new RecordBatch(schema, data);
});
return new ArrowTable(schema, batches);
}
/**
* Serialize an Arrow Table into a buffer using the Arrow IPC File serialization
*
@@ -951,7 +1304,10 @@ export async function fromTableToBuffer(
if (schema !== undefined && schema !== null) {
schema = sanitizeSchema(schema);
}
const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema);
const tableWithEmbeddings = materializeViewColumns(
await applyEmbeddings(table, embeddings, schema),
);
validateBlobSchema(tableWithEmbeddings.schema);
const writer = RecordBatchFileWriter.writeAll(tableWithEmbeddings);
return Buffer.from(await writer.toUint8Array());
}
@@ -1021,7 +1377,8 @@ export async function fromRecordBatchToBuffer(
export async function fromRecordBatchToStreamBuffer(
batch: RecordBatch,
): Promise<Buffer> {
const writer = RecordBatchStreamWriter.writeAll([batch]);
const table = materializeViewColumns(new ArrowTable([batch]));
const writer = RecordBatchStreamWriter.writeAll(table);
return Buffer.from(await writer.toUint8Array());
}
@@ -1038,7 +1395,9 @@ export async function fromTableToStreamBuffer(
embeddings?: EmbeddingFunctionConfig,
schema?: SchemaLike,
): Promise<Buffer> {
const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema);
const tableWithEmbeddings = materializeViewColumns(
await applyEmbeddings(table, embeddings, schema),
);
const writer = RecordBatchStreamWriter.writeAll(tableWithEmbeddings);
return Buffer.from(await writer.toUint8Array());
}
@@ -1165,7 +1524,7 @@ export function ensureNestedFieldsExist(
for (const field of schema.fields) {
if (field.name in row) {
if (
field.type.constructor.name === "Struct" &&
isPlainStructField(field) &&
row[field.name] !== null &&
row[field.name] !== undefined
) {
@@ -1182,10 +1541,9 @@ export function ensureNestedFieldsExist(
} else {
// Keep a missing struct valid while filling each of its children with
// null. This is distinct from an explicitly null struct value.
completeRow[field.name] =
field.type.constructor.name === "Struct"
? ensureStructFieldsExist({}, field.type as Struct)
: null;
completeRow[field.name] = isPlainStructField(field)
? ensureStructFieldsExist({}, field.type as Struct)
: null;
}
}
@@ -1193,6 +1551,15 @@ export function ensureNestedFieldsExist(
});
}
/**
* Blob fields are Arrow structs, but their values are bytes, URIs, or
* `{ data } | { uri }` inputs that the blob coercion in `makeArrowTable`
* handles, so they must not be filled in like ordinary structs.
*/
function isPlainStructField(field: Field): boolean {
return field.type.constructor.name === "Struct" && !isBlobField(field);
}
/**
* Recursively ensures that all fields in a struct type exist in the data,
* filling missing fields with null values.
@@ -1206,7 +1573,7 @@ function ensureStructFieldsExist(
for (const childField of structType.children) {
if (childField.name in data) {
if (
childField.type.constructor.name === "Struct" &&
isPlainStructField(childField) &&
data[childField.name] !== null &&
data[childField.name] !== undefined
) {
@@ -1222,10 +1589,9 @@ function ensureStructFieldsExist(
} else {
// Keep a missing struct valid while filling each of its children with
// null. This is distinct from an explicitly null struct value.
completeStruct[childField.name] =
childField.type.constructor.name === "Struct"
? ensureStructFieldsExist({}, childField.type as Struct)
: null;
completeStruct[childField.name] = isPlainStructField(childField)
? ensureStructFieldsExist({}, childField.type as Struct)
: null;
}
}
nodejs/lancedb/blob.ts
+481
View File
@@ -0,0 +1,481 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { Field, LargeBinary, Struct, Utf8 } from "apache-arrow";
import { BlobFile as NativeBlobFile } from "./native";
const BLOB_V2_EXTENSION_NAME = "lance.blob.v2";
const INLINE_SIZE_THRESHOLD_KEY = "lance-encoding:blob-inline-size-threshold";
const DEDICATED_SIZE_THRESHOLD_KEY =
"lance-encoding:blob-dedicated-size-threshold";
const PACK_FILE_SIZE_THRESHOLD_KEY =
"lance-encoding:blob-pack-file-size-threshold";
/**
* Bytes accepted for a blob value. `ArrayBuffer` is wrapped without copying.
* `Blob` and `File` are read with `arrayBuffer()`, which only the async write
* paths ({@link Connection.createTable}, {@link Table.add},
* {@link Table.mergeInsert}) can do.
*/
export type BlobData = Buffer | Uint8Array | ArrayBuffer | Blob;
/** A URI accepted for a blob value. A `URL` is stored as its `href`. */
export type BlobUri = string | URL;
/**
* A value accepted for a `lance.blob.v2` column (see {@link blob}).
*
* Either inline bytes, a URI pointing at external bytes, or a struct that sets
* exactly one of `data` or `uri`. `null` and `undefined` write a null blob.
*
* @example
* ```ts
* import { pathToFileURL } from "node:url";
* import type { BlobInput } from "@lancedb/lancedb";
*
* const rows: { id: bigint; image: BlobInput }[] = [
* { id: 1n, image: await (await fetch(url)).arrayBuffer() },
* { id: 2n, image: pathToFileURL("/data/cat.png") },
* { id: 3n, image: { data: new Blob(["hello"]) } },
* ];
* await table.add(rows);
* ```
*/
export type BlobInput =
| BlobData
| BlobUri
| { data: BlobData; uri?: null }
| { data?: null; uri: BlobUri }
| null
| undefined;
/** A blob value normalized to the columns of the blob struct. */
export type BlobValue = {
data: Buffer | Uint8Array | null;
uri: string | null;
};
export type BlobOptions = {
/** Defaults to true. */
nullable?: boolean;
/**
* Max payload bytes kept inline in the data file. Zero is allowed. Must be a
* safe integer.
*/
inlineSizeThreshold?: number;
/**
* Max payload bytes stored in a packed sidecar before a dedicated file. Must
* be a positive safe integer.
*/
dedicatedSizeThreshold?: number;
/**
* Max bytes in one packed sidecar before starting another. Must be a positive
* safe integer.
*/
packFileSizeThreshold?: number;
};
/**
* Declares a `lance.blob.v2` column.
*
* Query results are descriptors, not payload bytes. Use {@link Table.fetchBlobs}
* or {@link Table.fetchBlobFiles} to read bytes.
*
* @example
* ```ts
* import { readFile } from "node:fs/promises";
* import { Field, Int64, Schema } from "apache-arrow";
* import { blob, connect } from "@lancedb/lancedb";
*
* const db = await connect("./data");
* const video = await readFile("clip.mp4");
* const table = await db.createTable(
* "videos",
* [{ id: 1n, video }],
* {
* schema: new Schema([
* new Field("id", new Int64()),
* blob("video"),
* ]),
* },
* );
*
* const rows = await table.query().select(["id"]).withRowId().toArray();
* const rowIds = rows.map((row) => row._rowid as bigint);
* const bytes = await table.fetchBlobs("video", rowIds);
*
* const [handle] = await table.fetchBlobFiles("video", rowIds);
* const size = handle!.size();
* const header = await handle!.readRange(0n, size < 65536n ? size : 65536n);
* ```
*/
export function blob(name: string, options: BlobOptions = {}): Field {
const metadata = new Map<string, string>([
["ARROW:extension:name", BLOB_V2_EXTENSION_NAME],
]);
setThreshold(
metadata,
INLINE_SIZE_THRESHOLD_KEY,
"inlineSizeThreshold",
options.inlineSizeThreshold,
0,
);
setThreshold(
metadata,
DEDICATED_SIZE_THRESHOLD_KEY,
"dedicatedSizeThreshold",
options.dedicatedSizeThreshold,
1,
);
setThreshold(
metadata,
PACK_FILE_SIZE_THRESHOLD_KEY,
"packFileSizeThreshold",
options.packFileSizeThreshold,
1,
);
return new Field(
name,
new Struct([
new Field("data", new LargeBinary(), true),
new Field("uri", new Utf8(), true),
]),
options.nullable ?? true,
metadata,
);
}
/**
* Checks for the `lance.blob.v2` extension marker. Does not validate the
* field's storage type.
*/
export function isBlobField(field: Field): boolean {
return field.metadata?.get("ARROW:extension:name") === BLOB_V2_EXTENSION_NAME;
}
/**
* A lazy handle to blob bytes. Create one with {@link Table.fetchBlobFiles}.
*
* @hideconstructor
*/
export class BlobFile {
private readonly inner: NativeBlobFile;
private constructor(inner: NativeBlobFile) {
if (!(inner instanceof NativeBlobFile)) {
throw new Error("BlobFile handles come from Table.fetchBlobFiles");
}
this.inner = inner;
}
/** @ignore */
static fromNative(inner: NativeBlobFile): BlobFile {
return new BlobFile(inner);
}
/** Returns the blob size in bytes. */
size(): bigint {
return this.inner.size();
}
/**
* Reads from the cursor and advances the cursor.
*
* Reads to the end when `maxBytes` is omitted, or at most `maxBytes` bytes
* otherwise. Returns an empty buffer at the end of the blob.
* {@link BlobFile.readRange} does not move the cursor.
*/
read(maxBytes?: bigint, options?: BlobReadOptions): Promise<Buffer> {
return runWithSignal(options?.signal, (signal) =>
this.inner.read(maxBytes, signal),
);
}
/**
* Reads the half-open byte range `[start, end)`.
*
* Fails when `end` is past the blob size. Does not move the cursor.
*/
readRange(
start: bigint,
end: bigint,
options?: BlobReadOptions,
): Promise<Buffer> {
return runWithSignal(options?.signal, (signal) =>
this.inner.readRange(start, end, signal),
);
}
/**
* Reads several half-open byte ranges. Returns one buffer per range, in
* the order given.
*
* Fails when any `end` is past the blob size. Does not move the cursor.
*/
readRanges(
ranges: BlobRange[],
options?: BlobReadOptions,
): Promise<Buffer[]> {
return runWithSignal(options?.signal, (signal) =>
this.inner.readRanges(ranges, signal),
);
}
/** Moves the cursor to `position`, in bytes from the start of the blob. */
seek(position: bigint): Promise<void> {
return this.inner.seek(position);
}
/** Returns the cursor position in bytes. */
tell(): Promise<bigint> {
return this.inner.tell();
}
/**
* Releases the handle. Reads after `close()` fail. Calling it again does
* nothing.
*/
close(): Promise<void> {
return this.inner.close();
}
/** Returns true after {@link BlobFile.close}. */
isClosed(): boolean {
return this.inner.isClosed();
}
}
/** Options for blob reads. */
export type BlobReadOptions = {
/**
* Cancels the read. The call rejects with `signal.reason`, and the native
* read stops, including in-flight requests to a remote table.
*/
signal?: AbortSignal;
};
/** @ignore */
export async function runWithSignal<T>(
signal: AbortSignal | undefined,
run: (signal: AbortSignal | undefined) => Promise<T>,
): Promise<T> {
if (signal === undefined) {
return run(undefined);
}
signal.throwIfAborted();
// napi-rs replaces `onabort` on the signal it receives, so pass a private
// signal and forward the caller's abort to it.
const controller = new AbortController();
const onAbort = () => controller.abort();
signal.addEventListener("abort", onAbort, { once: true });
try {
return await run(controller.signal);
} catch (err) {
if (signal.aborted) {
throw signal.reason;
}
throw err;
} finally {
signal.removeEventListener("abort", onAbort);
}
}
/** A half-open byte range `[start, end)` for {@link BlobFile.readRanges}. */
export type BlobRange = {
start: bigint;
end: bigint;
};
/**
* Rewrites the synchronous-only widenings of {@link BlobInput} (`ArrayBuffer`,
* `URL`) into `Uint8Array` and URI strings, keeping the value's shape. Other
* values are returned unchanged and validated later by {@link coerceBlobValue}.
*
* Throws on `Blob` / `File`, which must be read into bytes first.
*/
export function normalizeBlobInput(value: unknown): unknown {
if (blobToRead(value) !== undefined) {
throw new Error(
"Blob and File values must be read asynchronously. Pass them to " +
"createTable, add, or mergeInsert, or convert them with " +
"`new Uint8Array(await value.arrayBuffer())`",
);
}
if (value instanceof ArrayBuffer) {
return new Uint8Array(value);
}
if (value instanceof URL) {
return value.href;
}
if (isBlobStruct(value)) {
// Read `data` and `uri` explicitly: spreading a class instance would drop
// fields defined as getters on its prototype.
const out: Record<string, unknown> = {
...value,
data: value.data,
uri: value.uri,
};
let changed = false;
if (value.data instanceof ArrayBuffer) {
out.data = new Uint8Array(value.data);
changed = true;
}
if (value.uri instanceof URL) {
out.uri = value.uri.href;
changed = true;
}
return changed ? out : value;
}
return value;
}
/**
* Returns the `Blob` / `File` a blob value needs read, either the value itself
* or its `data` field, or `undefined` when there is nothing to read.
*/
export function blobToRead(value: unknown): Blob | undefined {
if (isBlobLike(value)) {
return value;
}
if (isBlobStruct(value) && isBlobLike(value.data)) {
return value.data;
}
return undefined;
}
/**
* Replaces the `Blob` that {@link blobToRead} found in `value` with `bytes`,
* keeping the value's shape.
*/
export function withBlobBytes(value: unknown, bytes: Uint8Array): unknown {
if (isBlobLike(value)) {
return bytes;
}
const record = value as Record<string, unknown>;
return { ...record, uri: record.uri, data: bytes };
}
export function coerceBlobValue(input: unknown): BlobValue | null {
const value = normalizeBlobInput(input);
if (value == null) {
return null;
}
if (isBlobBytes(value)) {
return { data: value, uri: null };
}
if (ArrayBuffer.isView(value)) {
throw new Error(
"Blob data must be Buffer, Uint8Array, ArrayBuffer, or Blob",
);
}
if (typeof value === "string") {
if (value === "") {
throw new Error("Blob uri cannot be empty");
}
return { data: null, uri: value };
}
if (isBlobStruct(value)) {
const record = value;
if (!("data" in record) && !("uri" in record)) {
throw new Error(
"Blob struct values must include a 'data' or 'uri' field",
);
}
const unknownKeys = Object.keys(record).filter(
(key) => key !== "data" && key !== "uri",
);
if (unknownKeys.length > 0) {
throw new Error(
`Blob struct values only support 'data' and 'uri', got ${unknownKeys
.map((key) => `'${key}'`)
.join(", ")}`,
);
}
const uri = record.uri;
if (uri === "") {
throw new Error("Blob uri cannot be empty");
}
if (uri != null && typeof uri !== "string") {
throw new Error(
`Blob uri must be a string, URL, or null, got ${typeof uri}`,
);
}
const data = record.data;
if (data != null && !isBlobBytes(data)) {
throw new Error(
"Blob data must be Buffer, Uint8Array, ArrayBuffer, Blob, or null",
);
}
const bytes = (data as Buffer | Uint8Array | null | undefined) ?? null;
const uriValue = uri ?? null;
if ((bytes == null) === (uriValue == null)) {
throw new Error(
"Blob struct values must set exactly one of 'data' or 'uri'",
);
}
return { data: bytes, uri: uriValue };
}
throw new Error(
"Blob column values must be Buffer, Uint8Array, ArrayBuffer, Blob, a URI string or URL, null, or { data?, uri? }",
);
}
function isBlobLike(value: unknown): value is Blob {
return typeof Blob !== "undefined" && value instanceof Blob;
}
/**
* Whether `value` is a `{ data?, uri? }` struct: any plain object, or any
* other object with a `data` or `uri` field (such as a class instance), apart
* from the binary, `Blob`, and `URL` values that are blob inputs themselves.
*/
function isBlobStruct(value: unknown): value is Record<string, unknown> {
if (
typeof value !== "object" ||
value === null ||
Array.isArray(value) ||
ArrayBuffer.isView(value) ||
value instanceof ArrayBuffer ||
(typeof SharedArrayBuffer !== "undefined" &&
value instanceof SharedArrayBuffer) ||
isBlobLike(value) ||
value instanceof URL
) {
return false;
}
const prototype = Object.getPrototypeOf(value);
return (
prototype === Object.prototype ||
prototype === null ||
"data" in value ||
"uri" in value
);
}
function isBlobBytes(value: unknown): value is Buffer | Uint8Array {
return Buffer.isBuffer(value) || value instanceof Uint8Array;
}
function setThreshold(
metadata: Map<string, string>,
key: string,
optionName: string,
value: number | undefined,
minimum: number,
): void {
if (value === undefined) {
return;
}
if (!Number.isSafeInteger(value)) {
throw new Error(`${optionName} must be a safe integer`);
}
if (value < minimum) {
throw new Error(
minimum <= 0
? `${optionName} must be non-negative`
: `${optionName} must be positive`,
);
}
metadata.set(key, String(value));
}
nodejs/lancedb/catalog.ts
+103
View File
@@ -0,0 +1,103 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { Connection, LocalConnection } from "./connection";
import { HeaderProvider } from "./header";
import {
JsHeaderProvider,
ListDatabasesResponse,
Catalog as NativeCatalog,
CatalogOptions as NativeCatalogOptions,
} from "./native.js";
import { OAuthConfig } from "./oauth";
/** Options shared by a catalog and the database connections it returns. */
export interface CatalogOptions
extends Omit<NativeCatalogOptions, "oauthConfig"> {
oauthConfig?: OAuthConfig;
/** Called for each request to supply authentication headers. */
headerProvider?:
| HeaderProvider
| (() => Record<string, string> | Promise<Record<string, string>>);
}
export type { ListDatabasesResponse } from "./native.js";
/** A remote catalog manages databases through the server's root namespace. */
export class Catalog {
/** @hidden */
constructor(private readonly inner: NativeCatalog) {}
/** The root namespace endpoint. */
get uri(): string {
return this.inner.uri;
}
/** Create a database, or open an existing database when existOk is true. */
async createDatabase(
name: string,
options: { existOk?: boolean } = {},
): Promise<Connection> {
return new LocalConnection(
await this.inner.createDatabase(name, options.existOk),
);
}
/** Connect to an existing database by its logical name. */
async connectDatabase(name: string): Promise<Connection> {
return new LocalConnection(await this.inner.connectDatabase(name));
}
/** Drop an empty database. The server rejects nonempty databases. */
async dropDatabase(
name: string,
options: { ignoreMissing?: boolean } = {},
): Promise<void> {
await this.inner.dropDatabase(name, options.ignoreMissing);
}
/** List one page of databases; pass pageToken from a response for the next page. */
async listDatabases(
options: { limit?: number; pageToken?: string } = {},
): Promise<ListDatabasesResponse> {
if (
options.limit !== undefined &&
(!Number.isInteger(options.limit) ||
options.limit <= 0 ||
options.limit > 2147483647)
) {
throw new Error(
"Database list limit must be an integer between 1 and 2147483647",
);
}
return this.inner.listDatabases(options.limit, options.pageToken);
}
}
/**
* Connect to an HTTP(S) catalog endpoint. Catalog requests omit database-selection
* headers; opened database connections inherit authentication and client options.
*
* @example
* ```ts
* const catalog = await connectCatalog("https://my-server.example", { apiKey: "secret" });
* const db = await catalog.createDatabase("analytics", { existOk: true });
* const page = await catalog.listDatabases({ limit: 20 });
* ```
*/
export async function connectCatalog(
endpoint: string,
options: CatalogOptions = {},
): Promise<Catalog> {
const { headerProvider, ...nativeOptions } = options;
const provider = headerProvider
? new JsHeaderProvider(async () =>
typeof headerProvider === "function"
? headerProvider()
: headerProvider.getHeaders(),
)
: undefined;
return new Catalog(
await NativeCatalog.new(endpoint, nativeOptions, provider),
);
}
nodejs/lancedb/connection.ts
+166 -37
View File
@@ -1,7 +1,6 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { tableFromIPC } from "apache-arrow";
import {
Data,
SchemaLike,
@@ -9,6 +8,7 @@ import {
fromTableToStreamBuffer,
isArrowTable,
makeArrowTable,
resolveBlobInputs,
} from "./arrow";
import {
Table as ArrowTable,
@@ -16,6 +16,7 @@ import {
makeEmptyTable,
} from "./arrow";
import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
import { Job } from "./job";
import {
MaterializedView,
MaterializedViewSelect,
@@ -27,12 +28,11 @@ import type {
CreateNamespaceResponse,
DescribeNamespaceResponse,
DropNamespaceResponse,
Job,
JobDescription,
JobInfo,
ListNamespacesResponse,
ListTablesResponse,
} from "./native";
import { ViewDescription, viewDescriptionFromNative } from "./view";
export type {
CreateNamespaceResponse,
DescribeNamespaceResponse,
@@ -322,13 +322,13 @@ export abstract class Connection {
/**
* Define a materialized view named `name` over the table `source`.
*
* The view is created empty, with the query recorded in its schema
* metadata; `view.refresh()` computes the rows. The view is a normal
* table: it can be queried, indexed and searched, and it appears in
* `tableNames`. The source table must have stable row ids (create it with
* The view is populated before creation returns. Set `withNoData` to create
* only its definition and empty backing table. The view is a normal table:
* it can be queried, indexed and searched, and it appears in `tableNames`.
* The source table must have stable row ids (create it with
* the `newTableEnableStableRowIds` storage option); they keep the view's
* provenance valid across source compactions and cannot be enabled after
* a table exists. Local databases only.
* a table exists.
*/
abstract createMaterializedView(
name: string,
@@ -337,6 +337,7 @@ export abstract class Connection {
select?: MaterializedViewSelect;
where?: string;
limit?: number;
withNoData?: boolean;
},
): Promise<MaterializedView>;
@@ -354,6 +355,79 @@ export abstract class Connection {
*/
abstract listMaterializedViews(): Promise<string[]>;
/**
* Drop the materialized view named `name`.
*
* The view may become unavailable before physical cleanup finishes. Use
* {@link dropMaterializedViewAsync} to retain and wait for the cleanup job.
*
* Rejects a table that exists but is not a materialized view.
*/
abstract dropMaterializedView(
name: string,
namespacePath?: string[],
): Promise<void>;
/**
* Start dropping the materialized view named `name` and return its cleanup
* job without waiting for completion.
*
* Rejects a table that exists but is not a materialized view.
*/
abstract dropMaterializedViewAsync(
name: string,
namespacePath?: string[],
): Promise<Job>;
/**
* Create a view: a named query the database plans on every read.
*
* The query is planned once, at creation, so one that cannot be planned is
* rejected now rather than at the first read. A view holds no rows, and its
* readers see its sources as they are at read time.
*
* There is no replace: a name already taken is an error, and changing a
* view is a drop followed by a create.
*/
abstract createView(
name: string,
query: string,
namespacePath?: string[],
): Promise<ViewDescription>;
/**
* What this database records about the view named `name`: its defining
* query and the schema that query resolved to.
*/
abstract describeView(
name: string,
namespacePath?: string[],
): Promise<ViewDescription>;
/**
* Drop the view named `name` and wait for its definition to be deleted.
*
* The tables it reads are untouched: a view holds no rows of its own. Use
* {@link dropViewAsync} to retain the cleanup job instead of waiting on it.
*/
abstract dropView(name: string, namespacePath?: string[]): Promise<void>;
/**
* Start dropping the view named `name` and return the job deleting its
* definition, without waiting for completion.
*
* The name is free before this resolves. When nothing was bound to it, the
* returned job is already finished and has no id.
*/
abstract dropViewAsync(name: string, namespacePath?: string[]): Promise<Job>;
/**
* The names of the views in one namespace.
*
* Names only; a definition comes from {@link describeView}.
*/
abstract listViews(namespacePath?: string[]): Promise<string[]>;
abstract openTable(
name: string,
namespacePath?: string[],
@@ -557,24 +631,19 @@ export abstract class Connection {
): Promise<void>;
/**
* A {@link Job} handle for a server-side job by id.
* Open a server-side job by id, returning a handle with its record already
* populated. Rejects when the server has no such job, the way
* {@link Connection.openTable} does for a missing table.
*
* The handle is constructed without a server round trip; an unknown id
* surfaces when the handle is used. Dropping the handle has no effect on
* the job itself.
* The returned {@link Job} answers for its own state, specification,
* result, failure and event history, so there is no separate
* connection-level call for any of them.
*/
abstract job(jobId: string): Job;
abstract openJob(jobId: string): Promise<Job>;
/** List server-side jobs across the database's tables. */
abstract listJobs(): Promise<JobInfo[]>;
/**
* Describe a single server-side job by id.
*
* Resolves to `null` when the server has no such job.
*/
abstract getJob(jobId: string): Promise<JobDescription | null>;
/**
* Request cancellation of a server-side job by id.
*
@@ -584,11 +653,22 @@ export abstract class Connection {
abstract cancelJob(jobId: string): Promise<boolean>;
/**
* The lifecycle event history of a server-side job, as an Arrow table.
* Pause a server-side job by id.
*
* Lists history across all jobs when `jobId` is omitted.
* The job's workers drain and it stays parked until resumed. Resolves to
* "pausing", "already_paused", or "committing" -- a job finalizing its
* results cannot be parked; retry shortly.
*/
abstract jobHistory(jobId?: string): Promise<ArrowTable>;
abstract pauseJob(jobId: string): Promise<string>;
/**
* Resume a paused server-side job by id.
*
* Its workers pick their work back up from checkpoints. Resolves to
* "resumed", "still_pausing" -- the pause's worker drain is not confirmed
* yet; retry shortly -- or "not_paused".
*/
abstract resumeJob(jobId: string): Promise<string>;
}
/** @hideconstructor */
@@ -645,6 +725,7 @@ export class LocalConnection extends Connection {
select?: MaterializedViewSelect;
where?: string;
limit?: number;
withNoData?: boolean;
},
): Promise<MaterializedView> {
validateNonNegativeInteger(options?.limit, "limit");
@@ -654,6 +735,7 @@ export class LocalConnection extends Connection {
normalizeSelect(options?.select),
options?.where,
options?.limit,
options?.withNoData ?? false,
);
return new MaterializedView(new LocalTable(innerTable));
}
@@ -667,6 +749,53 @@ export class LocalConnection extends Connection {
return await this.inner.listMaterializedViews();
}
async dropMaterializedView(
name: string,
namespacePath?: string[],
): Promise<void> {
return this.inner.dropMaterializedView(name, namespacePath ?? []);
}
async dropMaterializedViewAsync(
name: string,
namespacePath?: string[],
): Promise<Job> {
return new Job(
await this.inner.dropMaterializedViewAsync(name, namespacePath ?? []),
);
}
async createView(
name: string,
query: string,
namespacePath?: string[],
): Promise<ViewDescription> {
return viewDescriptionFromNative(
await this.inner.createView(name, query, namespacePath ?? []),
);
}
async describeView(
name: string,
namespacePath?: string[],
): Promise<ViewDescription> {
return viewDescriptionFromNative(
await this.inner.describeView(name, namespacePath ?? []),
);
}
async dropView(name: string, namespacePath?: string[]): Promise<void> {
return this.inner.dropView(name, namespacePath ?? []);
}
async dropViewAsync(name: string, namespacePath?: string[]): Promise<Job> {
return new Job(await this.inner.dropViewAsync(name, namespacePath ?? []));
}
async listViews(namespacePath?: string[]): Promise<string[]> {
return this.inner.listViews(namespacePath ?? []);
}
async listTables(
namespacePathOrOptions?: string[] | Partial<ListTablesOptions>,
options?: Partial<ListTablesOptions>,
@@ -869,7 +998,7 @@ export class LocalConnection extends Connection {
}
async dropTableAsync(name: string, namespacePath?: string[]): Promise<Job> {
return this.inner.dropTableAsync(name, namespacePath ?? []);
return new Job(await this.inner.dropTableAsync(name, namespacePath ?? []));
}
async dropAllTables(namespacePath?: string[]): Promise<void> {
@@ -928,28 +1057,24 @@ export class LocalConnection extends Connection {
);
}
job(jobId: string): Job {
return this.inner.job(jobId);
async openJob(jobId: string): Promise<Job> {
return new Job(await this.inner.openJob(jobId));
}
async listJobs(): Promise<JobInfo[]> {
return this.inner.listJobs();
}
async getJob(jobId: string): Promise<JobDescription | null> {
return this.inner.getJob(jobId);
}
async cancelJob(jobId: string): Promise<boolean> {
return this.inner.cancelJob(jobId);
}
async jobHistory(jobId?: string): Promise<ArrowTable> {
const buf = await this.inner.jobHistory(jobId);
if (buf.length === 0) {
return new ArrowTable();
}
return tableFromIPC(buf);
async pauseJob(jobId: string): Promise<string> {
return this.inner.pauseJob(jobId);
}
async resumeJob(jobId: string): Promise<string> {
return this.inner.resumeJob(jobId);
}
}
@@ -1009,7 +1134,11 @@ async function parseTableData(
if (isArrowTable(data)) {
table = sanitizeTable(data);
} else {
table = makeArrowTable(data as Record<string, unknown>[], options);
const records = await resolveBlobInputs(
data as Record<string, unknown>[],
options?.schema,
);
table = makeArrowTable(records, options);
}
if (streaming) {
const buf = await fromTableToStreamBuffer(
nodejs/lancedb/index.ts
+36 -8
View File
@@ -26,6 +26,7 @@ export {
MaterializedViewDefinition,
MaterializedViewSelect,
} from "./materialized_view";
export { ViewDescription } from "./view";
export { JsHeaderProvider as NativeJsHeaderProvider } from "./native.js";
// OpenTelemetry metrics bridge. Only the high-level entry point is public; the
@@ -56,6 +57,10 @@ export {
AddResult,
AddColumnsResult,
RefreshColumnResult,
FunctionErrors,
FunctionErrorsOptions,
FunctionErrorRecord,
FunctionErrorFragment,
RefreshMaterializedViewResult,
AlterColumnsResult,
UpdateFieldMetadataResult,
@@ -72,11 +77,22 @@ export {
export {
makeArrowTable,
makeJsonField,
MakeArrowTableOptions,
Data,
VectorColumnOptions,
} from "./arrow";
export { blob, isBlobField, BlobFile } from "./blob";
export type {
BlobData,
BlobInput,
BlobOptions,
BlobRange,
BlobReadOptions,
BlobUri,
} from "./blob";
export {
Connection,
CreateTableOptions,
@@ -94,13 +110,9 @@ export {
RenameTableOptions,
} from "./connection";
export {
Job,
JobDescription,
JobFailureInfo,
JobInfo,
Session,
} from "./native.js";
export { JobFailureInfo, JobInfo, Session } from "./native.js";
export { Job, JobEventsOptions } from "./job";
export {
AutoQuery,
@@ -171,7 +183,15 @@ export {
TokenResponse,
} from "./header";
export { OAuthConfig, OAuthFlowType } from "./oauth";
export {
ClientAuthMethod,
OAuthConfig,
OAuthFlowType,
OAuthSession,
SessionLogout,
SessionStatus,
TokenCacheOptions,
} from "./oauth";
export { MergeInsertBuilder, WriteExecutionOptions } from "./merge";
@@ -182,6 +202,7 @@ export * as rerankers from "./rerankers";
export {
SchemaLike,
TableLike,
DataTypeLike,
FieldLike,
RecordBatchLike,
DataLike,
@@ -621,3 +642,10 @@ export async function connectNamespace(
);
return new LocalConnection(nativeConn);
}
export {
Catalog,
CatalogOptions,
ListDatabasesResponse,
connectCatalog,
} from "./catalog";
nodejs/lancedb/indices.ts
+3 -3
View File
@@ -26,7 +26,7 @@ export interface IvfPqOptions {
* This value controls how much the vector is compressed during the quantization step.
* The more sub vectors there are the less the vector is compressed. The default is
* the dimension of the vector divided by 16. If the dimension is not evenly divisible
* by 16 we use the dimension divded by 8.
* by 16 we use the dimension divided by 8.
*
* The above two cases are highly preferred. Having 8 or 16 values per subvector allows
* us to use efficient SIMD instructions.
@@ -228,7 +228,7 @@ export interface HnswPqOptions {
* This value controls how much the vector is compressed during the quantization step.
* The more sub vectors there are the less the vector is compressed. The default is
* the dimension of the vector divided by 16. If the dimension is not evenly divisible
* by 16 we use the dimension divded by 8.
* by 16 we use the dimension divided by 8.
*
* The above two cases are highly preferred. Having 8 or 16 values per subvector allows
* us to use efficient SIMD instructions.
@@ -825,7 +825,7 @@ export interface IndexOptions {
/**
* Advanced index configuration
*
* This option allows you to specify a specfic index to create and also
* This option allows you to specify a specific index to create and also
* allows you to pass in configuration for training the index.
*
* See the static methods on Index for details on the various index types.
nodejs/lancedb/job.ts
+188
View File
@@ -0,0 +1,188 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { Table as ArrowTable, tableFromIPC } from "apache-arrow";
import { JobFailureInfo, Job as NativeJob } from "./native";
/** Which of a job's events {@link Job.events} returns. */
export interface JobEventsOptions {
/** Maximum event rows to return, up to the server maximum of 10,000. */
limit?: number;
/** SQL-like filter over the event columns. */
filter?: string;
}
/**
* A handle to an operation that may still be running.
*
* The operation may already be complete when the handle is created.
*
* The detail getters read what the handle last observed. Submitting an
* operation returns only a job id, so populating them eagerly would cost an
* extra round trip on every call:
*
* - {@link Job.refresh} and {@link Job.status} fetch the whole record.
* - {@link Job.wait} records the terminal state it establishes, but not the
* rest of the record.
* - Everything is null until one of those runs.
*
* @hideconstructor
*/
export class Job {
private readonly inner: NativeJob;
constructor(inner: NativeJob) {
this.inner = inner;
}
/**
* Identifies the operation on the server that is running it.
*
* Operations that run in this process have no server id. The value is
* opaque: parsing it or storing it to resume the job later is not supported.
*/
get id(): string | null {
return this.inner.id ?? null;
}
/** The last observed lifecycle state, without contacting the backend. */
get state(): string | null {
return this.inner.state ?? null;
}
/**
* The job's type, as the server names it. Null for an in-process job, which
* has no server-side record.
*/
get jobType(): string | null {
return this.inner.jobType ?? null;
}
/** When the job was created, in milliseconds since the epoch. */
get creationMs(): number | null {
return this.inner.creationMs ?? null;
}
/** The job-type-specific specification it was submitted with. */
// biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
get spec(): any | null {
return parseJson(this.inner.specJson);
}
/**
* The job-type-specific terminal result. Null until the job succeeds, so a
* job that never terminates reports its progress through {@link Job.events}
* instead.
*/
// biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
get result(): any | null {
return parseJson(this.inner.resultJson);
}
/** Why the job failed, when it failed and the server reports a reason. */
get failure(): JobFailureInfo | null {
return this.inner.failure ?? null;
}
/**
* The operation's current lifecycle state: "running", "finished", "failed",
* or "cancelled".
*
* A point snapshot; unlike {@link Job.wait} it does not block or reject on a
* terminal failure state. Also refreshes the getters above.
*/
async status(): Promise<string> {
return this.inner.status();
}
/** Wait until the operation reaches a terminal state. */
async wait(): Promise<void> {
return this.inner.wait();
}
/** Request cancellation. Cancelling a finished operation is a no-op. */
async cancel(): Promise<void> {
return this.inner.cancel();
}
/**
* Ask the backend for this job's current state, and for a server-side job
* its full record, then cache it for the getters above.
*/
async refresh(): Promise<void> {
return this.inner.refresh();
}
/**
* This job's recorded lifecycle events.
*
* Where the getters above report a terminal result only once the job reaches
* one, events are written as the job runs and outlive the workers that
* produced them. A distributed job records a `claim`/`claim_complete` pair
* per unit of work, each carrying `rows_processed`, so a job that never
* finishes still accounts for what it did.
*
* The server caps results at 1000 rows by default and 10,000 at most, and
* truncates without saying so, so pass `limit` for a job that emits an event
* per fragment. `filter` is a SQL-like expression over the `state`,
* `updated_by`, `emitted_from`, `emitted_by`, and `claim_entity` columns.
*/
async events(options?: JobEventsOptions): Promise<ArrowTable> {
const buf = await this.inner.events(options?.limit, options?.filter);
if (buf.length === 0) {
return new ArrowTable();
}
return tableFromIPC(buf);
}
/**
* Every field the handle currently knows, one per line, with the JSON
* payloads indented -- a refresh job's spec and result are the point of
* printing it.
*/
toString(): string {
if (this.state === null) {
const known = this.id === null ? "" : `id=${JSON.stringify(this.id)}, `;
return `Job(${known}not refreshed)`;
}
const fields: string[] = [];
if (this.id !== null) {
fields.push(`id=${JSON.stringify(this.id)}`);
}
fields.push(`state=${JSON.stringify(this.state)}`);
if (this.jobType !== null) {
fields.push(`jobType=${JSON.stringify(this.jobType)}`);
}
if (this.creationMs !== null) {
fields.push(`creationMs=${this.creationMs}`);
}
for (const [name, value] of [
["spec", this.spec],
["result", this.result],
] as const) {
if (value !== null) {
fields.push(`${name}=${indentJson(value)}`);
}
}
if (this.failure !== null) {
fields.push(`failure=${indentJson(this.failure)}`);
}
return `Job(${fields.map((field) => `\n${REPR_INDENT}${field},`).join("")}\n)`;
}
[Symbol.for("nodejs.util.inspect.custom")](): string {
return this.toString();
}
}
const REPR_INDENT = " ";
// biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
function indentJson(value: any): string {
return JSON.stringify(value, null, 4).replace(/\n/g, `\n${REPR_INDENT}`);
}
// biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
function parseJson(raw: string | null | undefined): any | null {
return raw === null || raw === undefined ? null : JSON.parse(raw);
}
nodejs/lancedb/materialized_view.ts
+76 -31
View File
@@ -7,20 +7,22 @@ import { Table } from "./table";
/** Schema metadata key holding a materialized view's definition. */
export const DEFINITION_META_KEY = "mv.definition";
/** The query that defines a materialized view. */
/**
* The newest stored layout this version reads: `{"format": N, "query": "<SQL>"}`,
* format 2 being a query with `GROUP BY`, format 3 native pairs and format 4
* native dedup results.
*/
export const DEFINITION_FORMAT = 4;
/**
* The query that defines a materialized view, as stored:
* `SELECT columns FROM [ns.]table [, function(args) AS alias | , UNNEST(column) AS alias]
* [WHERE predicate] [GROUP BY expr, ...] [LIMIT n]`. A Function in `FROM`
* position yields one row per element it returns.
*/
export interface MaterializedViewDefinition {
/** Name of the source table, in the same database as the view. */
sourceTable: string;
/** `[output column, SQL expression]` pairs, in view schema order. */
projections: [string, string][];
/** SQL predicate selecting the source rows the view holds. */
filter?: string;
/** Cap on the number of rows the view holds. */
limit?: number;
/** Source columns the projections and filter read. */
inputs: string[];
/** Namespace holding the source table; empty is the root namespace. */
sourceNamespace: string[];
/** The defining query, in the canonical spelling the server stores. */
query: string;
}
/**
@@ -78,13 +80,31 @@ export function definitionFromMetadata(
if (raw === undefined) {
throw new Error(`Table '${name}' is not a materialized view`);
}
return definitionFromJson(raw, name);
}
/** @internal Parse the backend-independent definition returned by native code. */
export function definitionFromJson(
raw: string,
name: string,
): MaterializedViewDefinition {
// biome-ignore lint/suspicious/noExplicitAny: raw JSON
const value: any = JSON.parse(raw);
// "namespaced_select" keeps older readers from resolving the source at root.
if (value.format !== undefined) {
// A newer writer's layout is reported, never guessed at.
if (!Number.isInteger(value.format) || value.format > DEFINITION_FORMAT) {
throw new Error(
`materialized view '${name}' is stored in format ${value.format}, ` +
"which this version of lancedb cannot refresh",
);
}
return { query: value.query };
}
// The structured layout written before the format number.
if (value.kind !== "select" && value.kind !== "namespaced_select") {
throw new Error(
`materialized view '${name}' is defined by '${value.kind}', which this ` +
"version of lancedb cannot refresh",
`materialized view '${name}' is stored in format kind '${value.kind}', ` +
"which this version of lancedb cannot refresh",
);
}
const limit = value.limit ?? undefined;
@@ -96,18 +116,41 @@ export function definitionFromMetadata(
`materialized view '${name}' has a stored limit too large to represent exactly`,
);
}
return {
sourceTable: value.source_table,
// biome-ignore lint/suspicious/noExplicitAny: raw JSON
projections: (value.projections ?? []).map((p: any) => [
p.output,
p.expression,
]),
filter: value.filter ?? undefined,
limit,
inputs: value.inputs ?? [],
sourceNamespace: value.source_namespace ?? [],
};
return { query: legacyQuery(value, limit) };
}
function legacyIdent(name: string): string {
return /^[a-z_][a-z0-9_]*$/.test(name)
? name
: `\`${name.replace(/`/g, "``")}\``;
}
/** Render the pre-format structured layout as the query it described. */
// biome-ignore lint/suspicious/noExplicitAny: raw JSON
function legacyQuery(value: any, limit: number | undefined): string {
// biome-ignore lint/suspicious/noExplicitAny: raw JSON
const projections: any[] = value.projections ?? [];
const columns =
projections.length === 0
? "*"
: projections
.map((p) =>
p.expression === p.output || p.expression === `\`${p.output}\``
? p.expression
: `${p.expression} AS ${legacyIdent(p.output)}`,
)
.join(", ");
const table = [...(value.source_namespace ?? []), value.source_table]
.map(legacyIdent)
.join(".");
let query = `SELECT ${columns} FROM ${table}`;
if (value.filter !== undefined && value.filter !== null) {
query += ` WHERE ${value.filter}`;
}
if (limit !== undefined) {
query += ` LIMIT ${limit}`;
}
return query;
}
/**
@@ -134,10 +177,12 @@ export class MaterializedView {
return this.inner;
}
/** The query that defines the view, read from its stored schema. */
/** The query that defines the view. */
async definition(): Promise<MaterializedViewDefinition> {
const schema = await this.inner.schema();
return definitionFromMetadata(schema.metadata, this.name);
return definitionFromJson(
await this.inner.materializedViewDefinition(),
this.name,
);
}
/**
nodejs/lancedb/merge.ts
+1 -1
View File
@@ -27,7 +27,7 @@ export class MergeInsertBuilder {
* but that behavior is subject to change.
*
* An optional condition may be specified. If it is, then only
* matched rows that satisfy the condtion will be updated. Any
* matched rows that satisfy the condition will be updated. Any
* rows that do not satisfy the condition will be left as they
* are. Failing to satisfy the condition does not cause a
* "matched row" to become a "not matched" row.
nodejs/lancedb/oauth.ts
+242
View File
@@ -1,16 +1,79 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import {
OAuthConfig as NativeOAuthConfig,
OAuthSession as NativeOAuthSession,
} from "./native";
/**
* OAuth authentication flow types.
*/
export enum OAuthFlowType {
/** Client Credentials grant (service-to-service / M2M). */
ClientCredentials = "client_credentials",
/** Interactive Authorization Code grant, using PKCE by default. */
AuthorizationCode = "authorization_code",
/** Device Authorization grant for CLI and headless environments. */
DeviceCode = "device_code",
/** Azure Managed Identity via IMDS. */
AzureManagedIdentity = "azure_managed_identity",
}
/**
* Options for the persistent OAuth token cache.
*
* The cache is opt-in: it is only used when set as `tokenCache` on
* {@link OAuthConfig}. Only refresh tokens are persisted, in a private
* directory with owner-only permissions, so short-lived processes can reuse
* an authenticated session instead of re-prompting on every start.
*
* Multiple identities (issuer, client, scopes, resource, audience, flow, client authentication)
* get separate cache entries. Within one identity the most recent login wins.
*/
export interface TokenCacheOptions {
/**
* Directory that holds cached credentials. Defaults to
* `$XDG_CACHE_HOME/lancedb/oauth`, `$HOME/.cache/lancedb/oauth` on Unix,
* or `%LOCALAPPDATA%\\lancedb\\oauth` on Windows. The directory is created
* with owner-only permissions (`0700`) when missing.
*/
cacheDir?: string;
/**
* How long to wait for the cross-process refresh lock before failing, in
* seconds (default: 30).
*/
lockTimeoutSecs?: number;
}
/**
* How the client authenticates to the OAuth token endpoint.
*
* The method applies to every OAuth request that carries client
* authentication: client-credentials, authorization-code exchange,
* refresh-token, and device-authorization requests. The Azure managed
* identity flow ignores this option.
*/
export enum ClientAuthMethod {
/**
* No client authentication, for public clients using PKCE or the device
* flow. Cannot be combined with `clientSecret`.
*/
None = "none",
/**
* HTTP Basic authentication. This is the RFC 6749 recommended method and
* the normal default for confidential clients, including default Okta
* applications. Requires `clientSecret`.
*/
ClientSecretBasic = "client_secret_basic",
/**
* Credentials in the request body, for providers configured to require it.
* Requires `clientSecret`.
*/
ClientSecretPost = "client_secret_post",
}
/**
* OAuth configuration for LanceDB authentication.
*
@@ -31,6 +94,18 @@ export enum OAuthFlowType {
* };
* ```
*
* Providers requiring an explicit target can set `resource` and/or `audience`:
* ```typescript
* const targeted: OAuthConfig = {
* issuerUrl: "https://issuer.example.com",
* clientId: "app-id",
* clientSecret: "secret",
* scopes: ["read"],
* resource: "https://api.example.com",
* audience: "lancedb-api",
* };
* ```
*
* @example Azure Managed Identity:
* ```typescript
* const config: OAuthConfig = {
@@ -40,6 +115,21 @@ export enum OAuthFlowType {
* flow: OAuthFlowType.AzureManagedIdentity,
* };
* ```
*
* @example Authorization Code with PKCE:
* The authorization URL is written to stderr before LanceDB tries to open a
* browser, so it can be copied in headless environments.
* ```typescript
* const config: OAuthConfig = {
* issuerUrl: "https://login.microsoftonline.com/{tenant}/v2.0",
* clientId: "app-id",
* scopes: ["openid", "api://lancedb-api/access"],
* flow: OAuthFlowType.AuthorizationCode,
* };
* ```
*
* Device Authorization writes the verification URL and user code to stderr
* before polling begins.
*/
export interface OAuthConfig {
/**
@@ -58,12 +148,43 @@ export interface OAuthConfig {
*/
scopes: string[];
/**
* Resource indicator (RFC 8707), forwarded verbatim to authorization and token
* endpoints, including refresh requests. Must be an absolute URI without a
* fragment. Not supported for Azure managed identity.
*/
resource?: string;
/**
* Provider-specific audience, forwarded to authorization and token endpoints,
* including refresh requests. Not supported for Azure managed identity.
*/
audience?: string;
/** Authentication flow (default: ClientCredentials). */
flow?: OAuthFlowType;
/** Client secret (required for ClientCredentials). */
clientSecret?: string;
/**
* How the client authenticates to the token endpoint (default: auto).
* With a `clientSecret` the default is `ClientAuthMethod.ClientSecretBasic`,
* which matches the RFC 6749 recommendation and the default configuration
* of Okta confidential applications; without a secret the client is public
* and no client authentication is sent.
*/
clientAuthMethod?: ClientAuthMethod;
/** Loopback redirect URI for AuthorizationCode. */
redirectUri?: string;
/** Port for the AuthorizationCode loopback callback server (default: 8400). */
callbackPort?: number;
/** Protect AuthorizationCode with S256 PKCE (default: true). */
usePkce?: boolean;
/** Client ID for user-assigned managed identity (AzureManagedIdentity). */
managedIdentityClientId?: string;
@@ -73,4 +194,125 @@ export interface OAuthConfig {
* the TTL, each request refreshes the token.
*/
refreshBufferSecs?: number;
/**
* Opt in to the persistent token cache so short-lived processes reuse one
* session. Only refresh tokens are persisted. Only supported by
* AuthorizationCode and DeviceCode; Azure managed identity is rejected.
* Default: unset (memory only).
*/
tokenCache?: TokenCacheOptions;
}
/**
* Safe, non-secret view of a cached OAuth session, returned by
* {@link OAuthSession.status} and {@link OAuthSession.login}.
*/
export interface SessionStatus {
/**
* Whether a cached session exists that can obtain tokens without
* interactive authentication. Because access tokens are not persisted,
* this is `true` exactly when a refresh token is cached; the next
* connection refreshes with it rather than opening a browser or device
* prompt.
*/
refreshable: boolean;
/** Canonical issuer URL of the cached session. */
issuerUrl: string;
/** Client ID of the cached session. */
clientId: string;
/** Canonical (sorted, de-duplicated) scope set of the cached session. */
scopes: string[];
/** Resource indicator used to obtain the cached session, if configured. */
resource?: string;
/** Provider-specific audience used to obtain the cached session, if configured. */
audience?: string;
/** Flow that produced the cached session. */
flow: string;
/** When the cached session was obtained, as Unix seconds. */
obtainedAt?: number;
}
/** Result of {@link OAuthSession.logout}. */
export interface SessionLogout {
/**
* Whether a cached credential was removed. `false` means no matching
* session was cached; logout is idempotent.
*/
removed: boolean;
}
/**
* Explicit OAuth session lifecycle for the persistent token cache: eager
* `login`, non-secret `status`, and local `logout`.
*
* A session is built from the same {@link OAuthConfig} used to connect
* (including its `tokenCache` options). A connection created with the same
* configuration shares the cache, so logging in here prepares tokens for
* later processes without any database request.
*
* `login` always runs the configured interactive flow and replaces the cached
* session (the most recent login wins). `logout` removes only the local
* credential; it does not revoke anything with the provider and does not sign
* out of a browser SSO session.
*
* @example
* ```typescript
* const config: OAuthConfig = {
* issuerUrl: "https://issuer.example.com",
* clientId: "my-app",
* scopes: ["openid", "offline_access"],
* flow: OAuthFlowType.DeviceCode,
* tokenCache: { cacheDir: "/tmp/my-app/oauth-cache" },
* };
* const session = new OAuthSession(config);
* const status = await session.login();
* ```
*/
export class OAuthSession {
private readonly inner: NativeOAuthSession;
/** Create a session manager for the given OAuth configuration. */
constructor(config: OAuthConfig) {
this.inner = new NativeOAuthSession(config as unknown as NativeOAuthConfig);
}
/**
* Eagerly run the configured authentication flow and store the session.
*
* A successful login always replaces any prior cached session for this
* identity; if the provider does not issue a refresh token (for example
* without `offline_access`), the previous record is removed and the status
* reports `refreshable == false`.
*/
async login(): Promise<SessionStatus> {
return this.inner.login();
}
/**
* Report whether a matching cached session exists, with safe metadata.
*
* This never contacts the identity provider and never exposes token values.
*/
async status(): Promise<SessionStatus> {
return this.inner.status();
}
/**
* Remove the matching local cached credential.
*
* This only deletes the local cache entry. It does not revoke the refresh
* token with the provider and does not sign out of a browser SSO session.
* Repeated calls succeed; `removed` reports whether a credential existed.
*/
async logout(): Promise<SessionLogout> {
return this.inner.logout();
}
}
nodejs/lancedb/query.ts
+12 -23
View File
@@ -41,26 +41,6 @@ export async function* RecordBatchIterator(
}
}
class RecordBatchIterable<
NativeQueryType extends NativeQuery | NativeVectorQuery | NativeTakeQuery,
> implements AsyncIterable<RecordBatch>
{
private inner: NativeQueryType;
private options?: QueryExecutionOptions;
constructor(inner: NativeQueryType, options?: QueryExecutionOptions) {
this.inner = inner;
this.options = options;
}
// biome-ignore lint/suspicious/noExplicitAny: skip
[Symbol.asyncIterator](): AsyncIterator<RecordBatch<any>, any, undefined> {
return RecordBatchIterator(
this.inner.execute(this.options?.maxBatchLength, this.options?.timeoutMs),
);
}
}
/**
* Options that control the behavior of a particular query execution
*/
@@ -271,13 +251,20 @@ export class QueryBase<
return RecordBatchIterator(this.nativeExecute());
}
/** Collect the results as an Arrow @see {@link ArrowTable}. */
/**
* Collect the results as an Arrow @see {@link ArrowTable}.
*
* Empty results retain the query's output schema.
*/
async toArrow(options?: Partial<QueryExecutionOptions>): Promise<ArrowTable> {
const batches = [];
const inner = await this.getInner();
for await (const batch of new RecordBatchIterable(inner, options)) {
const iterator = this.nativeExecute(options);
for await (const batch of RecordBatchIterator(iterator)) {
batches.push(batch);
}
if (batches.length === 0) {
return tableFromIPC((await iterator).schema());
}
return new ArrowTable(batches);
}
@@ -532,6 +519,8 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
/**
* Set the number of partitions to search (probe)
*
* The number of probes must be greater than 0.
*
* This argument is only used when the vector column has an IVF PQ index.
* If there is no index then this value is ignored.
*
nodejs/lancedb/sanitize.ts
+138 -16
View File
@@ -3,12 +3,13 @@
// The utilities in this file help sanitize data from the user's arrow
// library into the types expected by vectordb's arrow library. Node
// generally allows for mulitple versions of the same library (and sometimes
// generally allows for multiple versions of the same library (and sometimes
// even multiple copies of the same version) to be installed at the same
// time. However, arrow-js uses instanceof which expected that the input
// comes from the exact same library instance. This is not always the case
// and so we must sanitize the input to ensure that it is compatible.
import * as arrowModule from "apache-arrow";
import { BufferType, Data, Vector } from "apache-arrow";
import type { IntBitWidth, TKeys, TimeBitWidth } from "apache-arrow/type";
import {
@@ -43,6 +44,8 @@ import {
Interval,
IntervalDayTime,
IntervalYearMonth,
LargeBinary,
LargeUtf8,
List,
Map_,
Null,
@@ -74,6 +77,66 @@ import {
Utf8,
} from "./arrow";
/**
* Whether `value` was created by *this* copy of apache-arrow's `ctor`.
*
* Arrow 21 overrides `Symbol.hasInstance` on `Schema`, `Field`, `Table`,
* `RecordBatch`, `Vector` and `Data` so that `instanceof` also matches objects
* built by a different copy of the library. Those are exactly the objects
* this module has to rebuild: the `DataType` instances nested inside them are
* deliberately *not* given that treatment upstream, so Arrow's own builders and
* writers reject them. Walk the real prototype chain instead of trusting
* `instanceof`.
*/
function isOwnInstance<T>(
value: unknown,
// biome-ignore lint/suspicious/noExplicitAny: matches any class constructor
ctor: abstract new (...args: any[]) => T,
): value is T {
return (
typeof value === "object" &&
value !== null &&
Object.prototype.isPrototypeOf.call(ctor.prototype, value)
);
}
/**
* Types added to Arrow JS after release 18, keyed by the `Type` enum value
* they use. The library compiles and runs against every Arrow release in the
* peer range, so these are looked up by name at runtime instead of imported:
* an older Arrow simply does not have the class.
*/
type NewerTypeName =
| "IntervalMonthDayNano"
| "LargeList"
| "BinaryView"
| "Utf8View";
const NEWER_TYPES: ReadonlyArray<{
name: NewerTypeName;
typeId: number;
since: number;
}> = [
{ name: "IntervalMonthDayNano", typeId: -31, since: 20 },
{ name: "LargeList", typeId: 21, since: 21 },
{ name: "BinaryView", typeId: 23, since: 21 },
{ name: "Utf8View", typeId: 24, since: 21 },
];
// biome-ignore lint/suspicious/noExplicitAny: constructor arity varies by type
type DataTypeConstructor = new (...args: any[]) => DataType;
function newerTypeConstructor(name: NewerTypeName): DataTypeConstructor {
const ctor = (arrowModule as unknown as Record<string, unknown>)[name];
if (typeof ctor !== "function") {
const since = NEWER_TYPES.find((t) => t.name === name)?.since;
throw new Error(
`The Arrow type ${name} requires apache-arrow ${since} or newer, ` +
"but the installed apache-arrow does not provide it",
);
}
return ctor as DataTypeConstructor;
}
type SanitizationContext = {
types: WeakMap<object, DataType>;
vectors: WeakMap<object, Vector>;
@@ -210,19 +273,37 @@ export function sanitizeList(typeLike: object) {
return sanitizeListWithContext(typeLike, createSanitizationContext());
}
function sanitizeListChildWithContext(
typeLike: object,
context: SanitizationContext,
typeName: string,
): Field {
if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
throw Error(
`Expected a ${typeName} type to have an array-like \`children\` property`,
);
}
if (typeLike.children.length !== 1) {
throw Error(`Expected a ${typeName} type to have exactly one child`);
}
return sanitizeFieldWithContext(typeLike.children[0], context);
}
function sanitizeListWithContext(
typeLike: object,
context: SanitizationContext,
) {
if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
throw Error(
"Expected a List type to have an array-like `children` property",
);
}
if (typeLike.children.length !== 1) {
throw Error("Expected a List type to have exactly one child");
}
return new List(sanitizeFieldWithContext(typeLike.children[0], context));
return new List(sanitizeListChildWithContext(typeLike, context, "List"));
}
function sanitizeLargeListWithContext(
typeLike: object,
context: SanitizationContext,
) {
const LargeList = newerTypeConstructor("LargeList");
return new LargeList(
sanitizeListChildWithContext(typeLike, context, "LargeList"),
);
}
export function sanitizeStruct(typeLike: object) {
@@ -449,11 +530,30 @@ function sanitizeTypeWithContext(
return type;
}
function sanitizeNewerTypeById(
typeLike: object,
typeId: number,
context: SanitizationContext,
): DataType | undefined {
const entry = NEWER_TYPES.find((t) => t.typeId === typeId);
if (entry === undefined) {
return undefined;
}
if (entry.name === "LargeList") {
return sanitizeLargeListWithContext(typeLike, context);
}
return new (newerTypeConstructor(entry.name))();
}
function sanitizeTypeById(
typeLike: object,
typeId: Type,
context: SanitizationContext,
): DataType {
const newer = sanitizeNewerTypeById(typeLike, typeId, context);
if (newer !== undefined) {
return newer;
}
switch (typeId) {
case Type.NONE:
throw Error("Received a Type with a typeId of NONE");
@@ -465,8 +565,12 @@ function sanitizeTypeById(
return sanitizeFloat(typeLike);
case Type.Binary:
return new Binary();
case Type.LargeBinary:
return new LargeBinary();
case Type.Utf8:
return new Utf8();
case Type.LargeUtf8:
return new LargeUtf8();
case Type.Bool:
return new Bool();
case Type.Decimal:
@@ -566,7 +670,7 @@ function sanitizeFieldWithContext(
fieldLike: unknown,
context: SanitizationContext,
): Field {
if (fieldLike instanceof Field) {
if (isOwnInstance(fieldLike, Field)) {
return fieldLike;
}
if (typeof fieldLike !== "object" || fieldLike === null) {
@@ -620,7 +724,7 @@ function sanitizeSchemaWithContext(
schemaLike: SchemaLike,
context: SanitizationContext,
): Schema {
if (schemaLike instanceof Schema) {
if (isOwnInstance(schemaLike, Schema)) {
return schemaLike;
}
if (typeof schemaLike !== "object" || schemaLike === null) {
@@ -647,7 +751,7 @@ function sanitizeSchemaWithContext(
}
export function sanitizeTable(tableLike: TableLike): Table {
if (tableLike instanceof Table) {
if (isOwnInstance(tableLike, Table)) {
return tableLike;
}
if (typeof tableLike !== "object" || tableLike === null) {
@@ -675,7 +779,7 @@ function sanitizeRecordBatch(
batchLike: RecordBatchLike,
context: SanitizationContext,
): RecordBatch {
if (batchLike instanceof RecordBatch) {
if (isOwnInstance(batchLike, RecordBatch)) {
return batchLike;
}
if (typeof batchLike !== "object" || batchLike === null) {
@@ -704,11 +808,17 @@ type DictionaryDataLike = DataLike & {
dictionary?: DictionaryVectorLike;
};
// Arrow 21 added `variadicBuffers` to `Data` for the Utf8View / BinaryView
// types. Older Arrow versions do not have the property, so it is optional.
type VariadicDataLike = DataLike & {
variadicBuffers?: ReadonlyArray<Uint8Array>;
};
function sanitizeData(
dataLike: DataLike,
context: SanitizationContext,
): Data<DataType> {
if (dataLike instanceof Data) {
if (isOwnInstance(dataLike, Data)) {
return dataLike;
}
const cachedData = context.data.get(dataLike);
@@ -726,7 +836,12 @@ function sanitizeData(
context.vectors.set(dictionaryLike, dictionary);
}
}
const data = new Data(
// Arrow 21 added an eighth constructor parameter (`variadicBuffers`); older
// releases ignore the extra argument but their typings do not declare it.
const DataCtor = Data as unknown as new (
...args: unknown[]
) => Data<DataType>;
const data = new DataCtor(
sanitizeTypeWithContext(dataLike.type, context),
dataLike.offset,
dataLike.length,
@@ -739,6 +854,7 @@ function sanitizeData(
},
dataLike.children.map((child) => sanitizeData(child, context)),
dictionary,
(dataLike as VariadicDataLike).variadicBuffers,
);
context.data.set(dataLike, data);
return data;
@@ -747,7 +863,11 @@ function sanitizeData(
const constructorsByTypeName = {
null: () => new Null(),
binary: () => new Binary(),
largebinary: () => new LargeBinary(),
binaryview: () => new (newerTypeConstructor("BinaryView"))(),
utf8: () => new Utf8(),
largeutf8: () => new LargeUtf8(),
utf8view: () => new (newerTypeConstructor("Utf8View"))(),
bool: () => new Bool(),
int8: () => new Int8(),
int16: () => new Int16(),
@@ -768,6 +888,8 @@ const constructorsByTypeName = {
timesecond: () => new TimeSecond(),
intervaldaytime: () => new IntervalDayTime(),
intervalyearmonth: () => new IntervalYearMonth(),
intervalmonthdaynano: () =>
new (newerTypeConstructor("IntervalMonthDayNano"))(),
durationnanosecond: () => new DurationNanosecond(),
durationmicrosecond: () => new DurationMicrosecond(),
durationmillisecond: () => new DurationMillisecond(),
nodejs/lancedb/schema.ts
+94 -12
View File
@@ -62,17 +62,25 @@ class SchemaInferrer {
}
private observe(path: string[], value: unknown, row: number): void {
assertSupportedValue(value, path.join("."), row);
const current = this.fields.get(path);
if (current === undefined) {
this.addField(path, value, row);
} else if (this.providedSchema === undefined) {
this.updateInferredField(path, value, row, current);
} else if (current instanceof DeferredTypeEvidence) {
if (value != null) {
this.addSchemaField(this.providedSchema, path, value, row);
}
} else if (current instanceof FieldTree || DataType.isStruct(current)) {
assertStructValue(path, value, row);
}
}
private addField(path: string[], value: unknown, row: number): void {
if (this.providedSchema !== undefined) {
this.addSchemaField(this.providedSchema, path, row);
this.addSchemaField(this.providedSchema, path, value, row);
return;
}
@@ -93,15 +101,31 @@ class SchemaInferrer {
}
}
private addSchemaField(schema: Schema, path: string[], row: number): void {
private addSchemaField(
schema: Schema,
path: string[],
value: unknown,
row: number,
): void {
const field = fieldAtPath(schema, path);
if (field === undefined) {
throw new Error(
`Found field not in schema: ${path.join(".")} at row ${row}`,
);
}
if (DataType.isStruct(field.type)) {
assertStructValue(path, value, row);
}
const conflict = this.fields.set(path, field.type);
// Nulls are kept as deferred evidence so that a struct field seen first as
// null can be replaced by a record in a later row, just as a record seen
// first already ignores later nulls.
const conflict = this.fields.set(
path,
DeferredTypeEvidence.from(value, row) ?? field.type,
(existing) =>
existing instanceof DeferredTypeEvidence && existing.isOnlyNulls(),
);
if (conflict !== undefined) {
throw branchConflictError(conflict, row, "Struct");
}
@@ -430,16 +454,74 @@ function* recordPathsAndValues(
}
function isRecord(value: unknown): value is Record<string, unknown> {
if (typeof value !== "object" || value === null) {
return false;
}
const prototype = Object.getPrototypeOf(value);
return (
typeof value === "object" &&
value !== null &&
!Array.isArray(value) &&
!(value instanceof RegExp) &&
!(value instanceof Date) &&
!(value instanceof Set) &&
!(value instanceof Map) &&
!(value instanceof Buffer) &&
!ArrayBuffer.isView(value)
(prototype === Object.prototype || prototype === null) &&
Object.keys(value).length > 0
);
}
function isUnsupportedObject(value: unknown): boolean {
if (
typeof value !== "object" ||
value === null ||
Array.isArray(value) ||
isRecord(value)
) {
return false;
}
// Preserve the object types already passed through to Arrow. Other objects
// must not disappear while walking struct fields.
return !(
value instanceof RegExp ||
value instanceof Date ||
value instanceof Set ||
value instanceof Map ||
Buffer.isBuffer(value) ||
ArrayBuffer.isView(value)
);
}
function assertSupportedValue(
value: unknown,
field: string,
row: number,
): void {
if (Array.isArray(value)) {
for (const [index, item] of value.entries()) {
assertSupportedValue(item, `${field}[${index}]`, row);
}
} else if (isRecord(value)) {
for (const [name, child] of Object.entries(value)) {
assertSupportedValue(child, `${field}.${name}`, row);
}
} else if (isUnsupportedObject(value)) {
throw new Error(
`Unsupported object value for field ${field} at row ${row}.`,
);
}
}
/**
* Rejects primitives, arrays, and typed arrays given for a struct field, which
* would otherwise be written as a struct with every child null.
*/
function assertStructValue(path: string[], value: unknown, row: number): void {
if (
value == null ||
(typeof value === "object" &&
!Array.isArray(value) &&
!ArrayBuffer.isView(value))
) {
return;
}
const kind =
typeof value === "object" ? value.constructor.name : typeof value;
throw new Error(
`Expected a struct value for field ${path.join(".")} at row ${row}, got ${kind}.`,
);
}

Some files were not shown because too many files have changed in this diff Show More