rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-25 05:02:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: update scoring autoprojection bindings for current main

Browse Source 
Merge main, reuse its existing query request flag and scanner handling, and remove the unused TypeScript error suppression. Cover local projections, deferred queries, and remote option forwarding.
This commit is contained in:
Xuanwo
2026-10-22 19:11:53 +02:00
parent a8f57e8043 9f0f6400bc
commit 7ead9361a0
1214 changed files with 329212 additions and 55117 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nodejs/CLAUDE.md → nodejs/AGENTS.md
+6 -6
View File
@@ -3,11 +3,11 @@ The core Rust library is in the `../rust/lancedb` directory, the rust binding
code is in the `src/` directory and the typescript bindings are in
the `lancedb/` directory.
Whenever you change the Rust code, you will need to recompile: `npm run build`.
Whenever you change the Rust code, you will need to recompile: `pnpm build`.
Common commands:
* Build: `npm run build`
* Lint: `npm run lint`
* Fix lints: `npm run lint-fix`
* Test: `npm test`
* Run single test file: `npm test __test__/arrow.test.ts`
* Build: `pnpm build`
* Lint: `pnpm lint`
* Fix lints: `pnpm lint-fix`
* Test: `pnpm test`
* Run single test file: `pnpm test __test__/arrow.test.ts`
nodejs/CONTRIBUTING.md
+16 -14
View File
@@ -1,7 +1,7 @@
# Contributing to LanceDB Typescript
This document outlines the process for contributing to LanceDB Typescript.
For general contribution guidelines, see [CONTRIBUTING.md](../CONTRIBUTING.md).
For general contribution guidelines, see [CONTRIBUTING.md](https://github.com/lancedb/lancedb/blob/main/CONTRIBUTING.md).
## Project layout
@@ -12,20 +12,22 @@ Typescript.
* `src/`: Rust bindings source code
* `lancedb/`: Typescript package source code
* `__test__/`: Unit tests
* `examples/`: An npm package with the examples shown in the documentation
* `examples/`: A pnpm package with the examples shown in the documentation
## Development environment
To set up your development environment, you will need to install the following:
1. Node.js 14 or later
2. Rust's package manager, Cargo. Use [rustup](https://rustup.rs/) to install.
3. [protoc](https://grpc.io/docs/protoc-installation/) (Protocol Buffers compiler)
1. Node.js 22 or later (required by pnpm 11)
2. [pnpm](https://pnpm.io/installation) 11 or later (or run via `corepack enable`,
which uses the `packageManager` field in `package.json`)
3. Rust's package manager, Cargo. Use [rustup](https://rustup.rs/) to install.
4. [protoc](https://grpc.io/docs/protoc-installation/) (Protocol Buffers compiler)
Initial setup:
```shell
npm install
pnpm install
```
### Commit Hooks
@@ -39,38 +41,38 @@ pre-commit install
## Development
Most common development commands can be run using the npm scripts.
Most common development commands can be run using the pnpm scripts.
Build the package
```shell
npm install
npm run build
pnpm install
pnpm build
```
Lint:
```shell
npm run lint
pnpm lint
```
Format and fix lints:
```shell
npm run lint-fix
pnpm lint-fix
```
Run tests:
```shell
npm test
pnpm test
```
To run a single test:
```shell
# Single file: table.test.ts
npm test -- table.test.ts
pnpm test -- table.test.ts
# Single test: 'merge insert' in table.test.ts
npm test -- table.test.ts --testNamePattern=merge\ insert
pnpm test -- table.test.ts --testNamePattern=merge\ insert
```
nodejs/Cargo.toml
+20 -11
View File
@@ -1,7 +1,8 @@
[package]
name = "lancedb-nodejs"
edition.workspace = true
version = "0.21.4-beta.1"
version = "0.41.0-beta.13"
publish = false
license.workspace = true
description.workspace = true
repository.workspace = true
@@ -15,27 +16,35 @@ crate-type = ["cdylib"]
async-trait.workspace = true
arrow-ipc.workspace = true
arrow-array.workspace = true
arrow-buffer.workspace = true
half.workspace = true
arrow-schema.workspace = true
env_logger.workspace = true
futures.workspace = true
lancedb = { path = "../rust/lancedb", default-features = false }
napi = { version = "2.16.8", default-features = false, features = [
lancedb.workspace = true
lance-namespace.workspace = true
napi = { version = "3.8.3", default-features = false, features = [
"napi9",
"async"
"async",
"chrono_date",
"serde-json",
] }
napi-derive = "2.16.4"
chrono.workspace = true
serde_json.workspace = true
napi-derive = "3.5.2"
# Prevent dynamic linking of lzma, which comes from datafusion
lzma-sys = { version = "*", features = ["static"] }
lzma-sys = { version = "0.1", features = ["static"] }
log.workspace = true
# Workaround for build failure until we can fix it.
aws-lc-sys = "=0.28.0"
aws-lc-rs = "=1.13.0"
# Pin to resolve build failures; update periodically for security patches.
# rustls >= 0.23.45 (RUSTSEC-2026-0285) needs aws-lc-rs >= 1.18.
aws-lc-sys = "=0.45.0"
aws-lc-rs = "=1.18.1"
[build-dependencies]
napi-build = "2.1"
napi-build = "2.3.1"
[features]
default = ["remote", "lancedb/default"]
default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface", "lancedb/goosefs", "lancedb/metrics-otel"]
fp16kernels = ["lancedb/fp16kernels"]
remote = ["lancedb/remote"]
nodejs/NODEJS_THIRD_PARTY_LICENSES.md
+676
View File
@@ -0,0 +1,676 @@
[@75lb/deep-merge@1.1.2](https://github.com/75lb/deep-merge) - MIT
[@aashutoshrathi/word-wrap@1.2.6](https://github.com/aashutoshrathi/word-wrap) - MIT
[@ampproject/remapping@2.2.1](https://github.com/ampproject/remapping) - Apache-2.0
[@aws-crypto/crc32@3.0.0](https://github.com/aws/aws-sdk-js-crypto-helpers) - Apache-2.0
[@aws-crypto/crc32c@3.0.0](https://github.com/aws/aws-sdk-js-crypto-helpers) - Apache-2.0
[@aws-crypto/ie11-detection@3.0.0](https://github.com/aws/aws-sdk-js-crypto-helpers) - Apache-2.0
[@aws-crypto/sha1-browser@3.0.0](https://github.com/aws/aws-sdk-js-crypto-helpers) - Apache-2.0
[@aws-crypto/sha256-browser@3.0.0](https://github.com/aws/aws-sdk-js-crypto-helpers) - Apache-2.0
[@aws-crypto/sha256-browser@5.2.0](https://github.com/aws/aws-sdk-js-crypto-helpers) - Apache-2.0
[@aws-crypto/sha256-js@3.0.0](https://github.com/aws/aws-sdk-js-crypto-helpers) - Apache-2.0
[@aws-crypto/sha256-js@5.2.0](https://github.com/aws/aws-sdk-js-crypto-helpers) - Apache-2.0
[@aws-crypto/supports-web-crypto@3.0.0](https://github.com/aws/aws-sdk-js-crypto-helpers) - Apache-2.0
[@aws-crypto/supports-web-crypto@5.2.0](https://github.com/aws/aws-sdk-js-crypto-helpers) - Apache-2.0
[@aws-crypto/util@3.0.0](https://github.com/aws/aws-sdk-js-crypto-helpers) - Apache-2.0
[@aws-crypto/util@5.2.0](https://github.com/aws/aws-sdk-js-crypto-helpers) - Apache-2.0
[@aws-sdk/client-dynamodb@3.602.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/client-kms@3.549.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/client-s3@3.550.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/client-sso-oidc@3.549.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/client-sso-oidc@3.600.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/client-sso@3.549.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/client-sso@3.598.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/client-sts@3.549.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/client-sts@3.600.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/core@3.549.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/core@3.598.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/credential-provider-env@3.535.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/credential-provider-env@3.598.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/credential-provider-http@3.535.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/credential-provider-http@3.598.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/credential-provider-ini@3.549.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/credential-provider-ini@3.598.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/credential-provider-node@3.549.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/credential-provider-node@3.600.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/credential-provider-process@3.535.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/credential-provider-process@3.598.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/credential-provider-sso@3.549.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/credential-provider-sso@3.598.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/credential-provider-web-identity@3.549.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/credential-provider-web-identity@3.598.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/endpoint-cache@3.572.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/middleware-bucket-endpoint@3.535.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/middleware-endpoint-discovery@3.598.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/middleware-expect-continue@3.535.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/middleware-flexible-checksums@3.535.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/middleware-host-header@3.535.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/middleware-host-header@3.598.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/middleware-location-constraint@3.535.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/middleware-logger@3.535.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/middleware-logger@3.598.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/middleware-recursion-detection@3.535.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/middleware-recursion-detection@3.598.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/middleware-sdk-s3@3.535.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/middleware-signing@3.535.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/middleware-ssec@3.537.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/middleware-user-agent@3.540.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/middleware-user-agent@3.598.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/region-config-resolver@3.535.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/region-config-resolver@3.598.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/signature-v4-multi-region@3.535.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/token-providers@3.549.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/token-providers@3.598.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/types@3.535.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/types@3.598.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/util-arn-parser@3.535.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/util-endpoints@3.540.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/util-endpoints@3.598.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/util-locate-window@3.535.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/util-user-agent-browser@3.535.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/util-user-agent-browser@3.598.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/util-user-agent-node@3.535.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/util-user-agent-node@3.598.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/util-utf8-browser@3.259.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@aws-sdk/xml-builder@3.535.0](https://github.com/aws/aws-sdk-js-v3) - Apache-2.0
[@babel/code-frame@7.26.2](https://github.com/babel/babel) - MIT
[@babel/compat-data@7.23.5](https://github.com/babel/babel) - MIT
[@babel/core@7.23.7](https://github.com/babel/babel) - MIT
[@babel/generator@7.23.6](https://github.com/babel/babel) - MIT
[@babel/helper-compilation-targets@7.23.6](https://github.com/babel/babel) - MIT
[@babel/helper-environment-visitor@7.22.20](https://github.com/babel/babel) - MIT
[@babel/helper-function-name@7.23.0](https://github.com/babel/babel) - MIT
[@babel/helper-hoist-variables@7.22.5](https://github.com/babel/babel) - MIT
[@babel/helper-module-imports@7.22.15](https://github.com/babel/babel) - MIT
[@babel/helper-module-transforms@7.23.3](https://github.com/babel/babel) - MIT
[@babel/helper-plugin-utils@7.22.5](https://github.com/babel/babel) - MIT
[@babel/helper-simple-access@7.22.5](https://github.com/babel/babel) - MIT
[@babel/helper-split-export-declaration@7.22.6](https://github.com/babel/babel) - MIT
[@babel/helper-string-parser@7.25.9](https://github.com/babel/babel) - MIT
[@babel/helper-validator-identifier@7.25.9](https://github.com/babel/babel) - MIT
[@babel/helper-validator-option@7.23.5](https://github.com/babel/babel) - MIT
[@babel/helpers@7.27.0](https://github.com/babel/babel) - MIT
[@babel/parser@7.27.0](https://github.com/babel/babel) - MIT
[@babel/plugin-syntax-async-generators@7.8.4](https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-async-generators) - MIT
[@babel/plugin-syntax-bigint@7.8.3](https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-bigint) - MIT
[@babel/plugin-syntax-class-properties@7.12.13](https://github.com/babel/babel) - MIT
[@babel/plugin-syntax-import-meta@7.10.4](https://github.com/babel/babel) - MIT
[@babel/plugin-syntax-json-strings@7.8.3](https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-json-strings) - MIT
[@babel/plugin-syntax-jsx@7.23.3](https://github.com/babel/babel) - MIT
[@babel/plugin-syntax-logical-assignment-operators@7.10.4](https://github.com/babel/babel) - MIT
[@babel/plugin-syntax-nullish-coalescing-operator@7.8.3](https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-nullish-coalescing-operator) - MIT
[@babel/plugin-syntax-numeric-separator@7.10.4](https://github.com/babel/babel) - MIT
[@babel/plugin-syntax-object-rest-spread@7.8.3](https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-object-rest-spread) - MIT
[@babel/plugin-syntax-optional-catch-binding@7.8.3](https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-optional-catch-binding) - MIT
[@babel/plugin-syntax-optional-chaining@7.8.3](https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-optional-chaining) - MIT
[@babel/plugin-syntax-top-level-await@7.14.5](https://github.com/babel/babel) - MIT
[@babel/plugin-syntax-typescript@7.23.3](https://github.com/babel/babel) - MIT
[@babel/template@7.27.0](https://github.com/babel/babel) - MIT
[@babel/traverse@7.23.7](https://github.com/babel/babel) - MIT
[@babel/types@7.27.0](https://github.com/babel/babel) - MIT
[@bcoe/v8-coverage@0.2.3](https://github.com/demurgos/v8-coverage) - MIT
[@biomejs/biome@1.8.3](https://github.com/biomejs/biome) - MIT OR Apache-2.0
[@biomejs/cli-darwin-arm64@1.8.3](https://github.com/biomejs/biome) - MIT OR Apache-2.0
[@eslint-community/eslint-utils@4.4.0](https://github.com/eslint-community/eslint-utils) - MIT
[@eslint-community/regexpp@4.10.0](https://github.com/eslint-community/regexpp) - MIT
[@eslint/eslintrc@2.1.4](https://github.com/eslint/eslintrc) - MIT
[@eslint/js@8.57.0](https://github.com/eslint/eslint) - MIT
[@huggingface/jinja@0.3.2](https://github.com/huggingface/huggingface.js) - MIT
[@huggingface/transformers@3.0.2](https://github.com/huggingface/transformers.js) - Apache-2.0
[@humanwhocodes/config-array@0.11.14](https://github.com/humanwhocodes/config-array) - Apache-2.0
[@humanwhocodes/module-importer@1.0.1](https://github.com/humanwhocodes/module-importer) - Apache-2.0
[@humanwhocodes/object-schema@2.0.2](https://github.com/humanwhocodes/object-schema) - BSD-3-Clause
[@img/sharp-darwin-arm64@0.33.5](https://github.com/lovell/sharp) - Apache-2.0
[@img/sharp-libvips-darwin-arm64@1.0.4](https://github.com/lovell/sharp-libvips) - LGPL-3.0-or-later
[@isaacs/cliui@8.0.2](https://github.com/yargs/cliui) - ISC
[@isaacs/fs-minipass@4.0.1](https://github.com/npm/fs-minipass) - ISC
[@istanbuljs/load-nyc-config@1.1.0](https://github.com/istanbuljs/load-nyc-config) - ISC
[@istanbuljs/schema@0.1.3](https://github.com/istanbuljs/schema) - MIT
[@jest/console@29.7.0](https://github.com/jestjs/jest) - MIT
[@jest/core@29.7.0](https://github.com/jestjs/jest) - MIT
[@jest/environment@29.7.0](https://github.com/jestjs/jest) - MIT
[@jest/expect-utils@29.7.0](https://github.com/jestjs/jest) - MIT
[@jest/expect@29.7.0](https://github.com/jestjs/jest) - MIT
[@jest/fake-timers@29.7.0](https://github.com/jestjs/jest) - MIT
[@jest/globals@29.7.0](https://github.com/jestjs/jest) - MIT
[@jest/reporters@29.7.0](https://github.com/jestjs/jest) - MIT
[@jest/schemas@29.6.3](https://github.com/jestjs/jest) - MIT
[@jest/source-map@29.6.3](https://github.com/jestjs/jest) - MIT
[@jest/test-result@29.7.0](https://github.com/jestjs/jest) - MIT
[@jest/test-sequencer@29.7.0](https://github.com/jestjs/jest) - MIT
[@jest/transform@29.7.0](https://github.com/jestjs/jest) - MIT
[@jest/types@29.6.3](https://github.com/jestjs/jest) - MIT
[@jridgewell/gen-mapping@0.3.3](https://github.com/jridgewell/gen-mapping) - MIT
[@jridgewell/resolve-uri@3.1.1](https://github.com/jridgewell/resolve-uri) - MIT
[@jridgewell/set-array@1.1.2](https://github.com/jridgewell/set-array) - MIT
[@jridgewell/sourcemap-codec@1.4.15](https://github.com/jridgewell/sourcemap-codec) - MIT
[@jridgewell/trace-mapping@0.3.22](https://github.com/jridgewell/trace-mapping) - MIT
[@lancedb/lancedb@0.26.2](https://github.com/lancedb/lancedb) - Apache-2.0
[@napi-rs/cli@2.18.3](https://github.com/napi-rs/napi-rs) - MIT
[@nodelib/fs.scandir@2.1.5](https://github.com/nodelib/nodelib/tree/master/packages/fs/fs.scandir) - MIT
[@nodelib/fs.stat@2.0.5](https://github.com/nodelib/nodelib/tree/master/packages/fs/fs.stat) - MIT
[@nodelib/fs.walk@1.2.8](https://github.com/nodelib/nodelib/tree/master/packages/fs/fs.walk) - MIT
[@pkgjs/parseargs@0.11.0](https://github.com/pkgjs/parseargs) - MIT
[@protobufjs/aspromise@1.1.2](https://github.com/dcodeIO/protobuf.js) - BSD-3-Clause
[@protobufjs/base64@1.1.2](https://github.com/dcodeIO/protobuf.js) - BSD-3-Clause
[@protobufjs/codegen@2.0.4](https://github.com/dcodeIO/protobuf.js) - BSD-3-Clause
[@protobufjs/eventemitter@1.1.0](https://github.com/dcodeIO/protobuf.js) - BSD-3-Clause
[@protobufjs/fetch@1.1.0](https://github.com/dcodeIO/protobuf.js) - BSD-3-Clause
[@protobufjs/float@1.0.2](https://github.com/dcodeIO/protobuf.js) - BSD-3-Clause
[@protobufjs/inquire@1.1.0](https://github.com/dcodeIO/protobuf.js) - BSD-3-Clause
[@protobufjs/path@1.1.2](https://github.com/dcodeIO/protobuf.js) - BSD-3-Clause
[@protobufjs/pool@1.1.0](https://github.com/dcodeIO/protobuf.js) - BSD-3-Clause
[@protobufjs/utf8@1.1.0](https://github.com/dcodeIO/protobuf.js) - BSD-3-Clause
[@shikijs/core@1.10.3](https://github.com/shikijs/shiki) - MIT
[@sinclair/typebox@0.27.8](https://github.com/sinclairzx81/typebox) - MIT
[@sinonjs/commons@3.0.1](https://github.com/sinonjs/commons) - BSD-3-Clause
[@sinonjs/fake-timers@10.3.0](https://github.com/sinonjs/fake-timers) - BSD-3-Clause
[@smithy/abort-controller@2.2.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/abort-controller@3.1.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/chunked-blob-reader-native@2.2.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/chunked-blob-reader@2.2.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/config-resolver@2.2.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/config-resolver@3.0.3](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/core@1.4.2](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/core@2.2.3](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/credential-provider-imds@2.3.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/credential-provider-imds@3.1.2](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/eventstream-codec@2.2.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/eventstream-serde-browser@2.2.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/eventstream-serde-config-resolver@2.2.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/eventstream-serde-node@2.2.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/eventstream-serde-universal@2.2.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/fetch-http-handler@2.5.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/fetch-http-handler@3.1.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/hash-blob-browser@2.2.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/hash-node@2.2.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/hash-node@3.0.2](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/hash-stream-node@2.2.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/invalid-dependency@2.2.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/invalid-dependency@3.0.2](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/is-array-buffer@2.2.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/is-array-buffer@3.0.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/md5-js@2.2.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/middleware-content-length@2.2.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/middleware-content-length@3.0.2](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/middleware-endpoint@2.5.1](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/middleware-endpoint@3.0.3](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/middleware-retry@2.3.1](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/middleware-retry@3.0.6](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/middleware-serde@2.3.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/middleware-serde@3.0.2](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/middleware-stack@2.2.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/middleware-stack@3.0.2](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/node-config-provider@2.3.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/node-config-provider@3.1.2](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/node-http-handler@2.5.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/node-http-handler@3.1.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/property-provider@2.2.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/property-provider@3.1.2](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/protocol-http@3.3.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/protocol-http@4.0.2](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/querystring-builder@2.2.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/querystring-builder@3.0.2](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/querystring-parser@2.2.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/querystring-parser@3.0.2](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/service-error-classification@2.1.5](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/service-error-classification@3.0.2](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/shared-ini-file-loader@2.4.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/shared-ini-file-loader@3.1.2](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/signature-v4@2.2.1](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/signature-v4@3.1.1](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/smithy-client@2.5.1](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/smithy-client@3.1.4](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/types@2.12.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/types@3.2.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/url-parser@2.2.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/url-parser@3.0.2](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/util-base64@2.3.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/util-base64@3.0.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/util-body-length-browser@2.2.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/util-body-length-browser@3.0.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/util-body-length-node@2.3.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/util-body-length-node@3.0.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/util-buffer-from@2.2.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/util-buffer-from@3.0.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/util-config-provider@2.3.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/util-config-provider@3.0.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/util-defaults-mode-browser@2.2.1](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/util-defaults-mode-browser@3.0.6](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/util-defaults-mode-node@2.3.1](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/util-defaults-mode-node@3.0.6](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/util-endpoints@1.2.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/util-endpoints@2.0.3](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/util-hex-encoding@2.2.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/util-hex-encoding@3.0.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/util-middleware@2.2.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/util-middleware@3.0.2](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/util-retry@2.2.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/util-retry@3.0.2](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/util-stream@2.2.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/util-stream@3.0.4](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/util-uri-escape@2.2.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/util-uri-escape@3.0.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/util-utf8@2.3.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/util-utf8@3.0.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/util-waiter@2.2.0](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@smithy/util-waiter@3.1.1](https://github.com/awslabs/smithy-typescript) - Apache-2.0
[@swc/helpers@0.5.12](https://github.com/swc-project/swc) - Apache-2.0
[@types/axios@0.14.0](https://github.com/mzabriskie/axios) - MIT
[@types/babel__core@7.20.5](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/babel__generator@7.6.8](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/babel__template@7.4.4](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/babel__traverse@7.20.5](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/command-line-args@5.2.3](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/command-line-usage@5.0.2](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/command-line-usage@5.0.4](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/graceful-fs@4.1.9](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/hast@3.0.4](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/istanbul-lib-coverage@2.0.6](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/istanbul-lib-report@3.0.3](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/istanbul-reports@3.0.4](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/jest@29.5.12](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/json-schema@7.0.15](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/node-fetch@2.6.11](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/node@18.19.26](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/node@20.16.10](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/node@20.17.9](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/node@22.7.4](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/node@25.9.8](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/semver@7.5.6](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/stack-utils@2.0.3](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/tmp@0.2.6](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/unist@3.0.2](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/yargs-parser@21.0.3](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/yargs@17.0.32](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@typescript-eslint/eslint-plugin@7.1.0](https://github.com/typescript-eslint/typescript-eslint) - MIT
[@typescript-eslint/parser@7.1.0](https://github.com/typescript-eslint/typescript-eslint) - BSD-2-Clause
[@typescript-eslint/scope-manager@7.1.0](https://github.com/typescript-eslint/typescript-eslint) - MIT
[@typescript-eslint/type-utils@7.1.0](https://github.com/typescript-eslint/typescript-eslint) - MIT
[@typescript-eslint/types@7.1.0](https://github.com/typescript-eslint/typescript-eslint) - MIT
[@typescript-eslint/typescript-estree@7.1.0](https://github.com/typescript-eslint/typescript-eslint) - BSD-2-Clause
[@typescript-eslint/utils@7.1.0](https://github.com/typescript-eslint/typescript-eslint) - MIT
[@typescript-eslint/visitor-keys@7.1.0](https://github.com/typescript-eslint/typescript-eslint) - MIT
[@ungap/structured-clone@1.2.0](https://github.com/ungap/structured-clone) - ISC
[abort-controller@3.0.0](https://github.com/mysticatea/abort-controller) - MIT
[acorn-jsx@5.3.2](https://github.com/acornjs/acorn-jsx) - MIT
[acorn@8.11.3](https://github.com/acornjs/acorn) - MIT
[agentkeepalive@4.5.0](https://github.com/node-modules/agentkeepalive) - MIT
[ajv@6.12.6](https://github.com/ajv-validator/ajv) - MIT
[ansi-escapes@4.3.2](https://github.com/sindresorhus/ansi-escapes) - MIT
[ansi-regex@5.0.1](https://github.com/chalk/ansi-regex) - MIT
[ansi-regex@6.1.0](https://github.com/chalk/ansi-regex) - MIT
[ansi-styles@4.3.0](https://github.com/chalk/ansi-styles) - MIT
[ansi-styles@5.2.0](https://github.com/chalk/ansi-styles) - MIT
[ansi-styles@6.2.1](https://github.com/chalk/ansi-styles) - MIT
[anymatch@3.1.3](https://github.com/micromatch/anymatch) - ISC
[apache-arrow@15.0.0](https://github.com/apache/arrow) - Apache-2.0
[apache-arrow@16.0.0](https://github.com/apache/arrow) - Apache-2.0
[apache-arrow@17.0.0](https://github.com/apache/arrow) - Apache-2.0
[apache-arrow@18.0.0](https://github.com/apache/arrow) - Apache-2.0
[apache-arrow@19.0.1](https://github.com/apache/arrow) - Apache-2.0
[apache-arrow@20.0.0](https://github.com/apache/arrow) - Apache-2.0
[apache-arrow@21.2.0](https://github.com/apache/arrow-js) - Apache-2.0
[argparse@1.0.10](https://github.com/nodeca/argparse) - MIT
[argparse@2.0.1](https://github.com/nodeca/argparse) - Python-2.0
[array-back@3.1.0](https://github.com/75lb/array-back) - MIT
[array-back@6.2.2](https://github.com/75lb/array-back) - MIT
[array-union@2.1.0](https://github.com/sindresorhus/array-union) - MIT
[asynckit@0.4.0](https://github.com/alexindigo/asynckit) - MIT
[axios@1.8.4](https://github.com/axios/axios) - MIT
[babel-jest@29.7.0](https://github.com/jestjs/jest) - MIT
[babel-plugin-istanbul@6.1.1](https://github.com/istanbuljs/babel-plugin-istanbul) - BSD-3-Clause
[babel-plugin-jest-hoist@29.6.3](https://github.com/jestjs/jest) - MIT
[babel-preset-current-node-syntax@1.0.1](https://github.com/nicolo-ribaudo/babel-preset-current-node-syntax) - MIT
[babel-preset-jest@29.6.3](https://github.com/jestjs/jest) - MIT
[balanced-match@1.0.2](https://github.com/juliangruber/balanced-match) - MIT
[base-64@0.1.0](https://github.com/mathiasbynens/base64) - MIT
[bowser@2.11.0](https://github.com/lancedikson/bowser) - MIT
[brace-expansion@1.1.11](https://github.com/juliangruber/brace-expansion) - MIT
[brace-expansion@2.0.1](https://github.com/juliangruber/brace-expansion) - MIT
[braces@3.0.3](https://github.com/micromatch/braces) - MIT
[browserslist@4.22.2](https://github.com/browserslist/browserslist) - MIT
[bs-logger@0.2.6](https://github.com/huafu/bs-logger) - MIT
[bser@2.1.1](https://github.com/facebook/watchman) - Apache-2.0
[buffer-from@1.1.2](https://github.com/LinusU/buffer-from) - MIT
[callsites@3.1.0](https://github.com/sindresorhus/callsites) - MIT
[camelcase@5.3.1](https://github.com/sindresorhus/camelcase) - MIT
[camelcase@6.3.0](https://github.com/sindresorhus/camelcase) - MIT
[caniuse-lite@1.0.30001579](https://github.com/browserslist/caniuse-lite) - CC-BY-4.0
[chalk-template@0.4.0](https://github.com/chalk/chalk-template) - MIT
[chalk@4.1.2](https://github.com/chalk/chalk) - MIT
[char-regex@1.0.2](https://github.com/Richienb/char-regex) - MIT
[charenc@0.0.2](https://github.com/pvorb/node-charenc) - BSD-3-Clause
[chownr@3.0.0](https://github.com/isaacs/chownr) - BlueOak-1.0.0
[ci-info@3.9.0](https://github.com/watson/ci-info) - MIT
[cjs-module-lexer@1.2.3](https://github.com/nodejs/cjs-module-lexer) - MIT
[cliui@8.0.1](https://github.com/yargs/cliui) - ISC
[co@4.6.0](https://github.com/tj/co) - MIT
[collect-v8-coverage@1.0.2](https://github.com/SimenB/collect-v8-coverage) - MIT
[color-convert@2.0.1](https://github.com/Qix-/color-convert) - MIT
[color-name@1.1.4](https://github.com/colorjs/color-name) - MIT
[color-string@1.9.1](https://github.com/Qix-/color-string) - MIT
[color@4.2.3](https://github.com/Qix-/color) - MIT
[combined-stream@1.0.8](https://github.com/felixge/node-combined-stream) - MIT
[command-line-args@5.2.1](https://github.com/75lb/command-line-args) - MIT
[command-line-args@6.0.2](https://github.com/75lb/command-line-args) - MIT
[command-line-usage@7.0.1](https://github.com/75lb/command-line-usage) - MIT
[concat-map@0.0.1](https://github.com/substack/node-concat-map) - MIT
[convert-source-map@2.0.0](https://github.com/thlorenz/convert-source-map) - MIT
[create-jest@29.7.0](https://github.com/jestjs/jest) - MIT
[cross-spawn@7.0.6](https://github.com/moxystudio/node-cross-spawn) - MIT
[crypt@0.0.2](https://github.com/pvorb/node-crypt) - BSD-3-Clause
[debug@4.3.4](https://github.com/debug-js/debug) - MIT
[dedent@1.5.1](https://github.com/dmnd/dedent) - MIT
[deep-is@0.1.4](https://github.com/thlorenz/deep-is) - MIT
[deepmerge@4.3.1](https://github.com/TehShrike/deepmerge) - MIT
[delayed-stream@1.0.0](https://github.com/felixge/node-delayed-stream) - MIT
[detect-libc@2.0.3](https://github.com/lovell/detect-libc) - Apache-2.0
[detect-newline@3.1.0](https://github.com/sindresorhus/detect-newline) - MIT
[diff-sequences@29.6.3](https://github.com/jestjs/jest) - MIT
[digest-fetch@1.3.0](https://github.com/devfans/digest-fetch) - ISC
[dir-glob@3.0.1](https://github.com/kevva/dir-glob) - MIT
[doctrine@3.0.0](https://github.com/eslint/doctrine) - Apache-2.0
[eastasianwidth@0.2.0](https://github.com/komagata/eastasianwidth) - MIT
[electron-to-chromium@1.4.642](https://github.com/kilian/electron-to-chromium) - ISC
[emittery@0.13.1](https://github.com/sindresorhus/emittery) - MIT
[emoji-regex@8.0.0](https://github.com/mathiasbynens/emoji-regex) - MIT
[emoji-regex@9.2.2](https://github.com/mathiasbynens/emoji-regex) - MIT
[entities@4.5.0](https://github.com/fb55/entities) - BSD-2-Clause
[error-ex@1.3.2](https://github.com/qix-/node-error-ex) - MIT
[escalade@3.1.1](https://github.com/lukeed/escalade) - MIT
[escape-string-regexp@2.0.0](https://github.com/sindresorhus/escape-string-regexp) - MIT
[escape-string-regexp@4.0.0](https://github.com/sindresorhus/escape-string-regexp) - MIT
[eslint-scope@7.2.2](https://github.com/eslint/eslint-scope) - BSD-2-Clause
[eslint-visitor-keys@3.4.3](https://github.com/eslint/eslint-visitor-keys) - Apache-2.0
[eslint@8.57.0](https://github.com/eslint/eslint) - MIT
[espree@9.6.1](https://github.com/eslint/espree) - BSD-2-Clause
[esprima@4.0.1](https://github.com/jquery/esprima) - BSD-2-Clause
[esquery@1.5.0](https://github.com/estools/esquery) - BSD-3-Clause
[esrecurse@4.3.0](https://github.com/estools/esrecurse) - BSD-2-Clause
[estraverse@5.3.0](https://github.com/estools/estraverse) - BSD-2-Clause
[esutils@2.0.3](https://github.com/estools/esutils) - BSD-2-Clause
[event-target-shim@5.0.1](https://github.com/mysticatea/event-target-shim) - MIT
[execa@5.1.1](https://github.com/sindresorhus/execa) - MIT
[exit@0.1.2](https://github.com/cowboy/node-exit) - MIT
[expect@29.7.0](https://github.com/jestjs/jest) - MIT
[fast-deep-equal@3.1.3](https://github.com/epoberezkin/fast-deep-equal) - MIT
[fast-glob@3.3.2](https://github.com/mrmlnc/fast-glob) - MIT
[fast-json-stable-stringify@2.1.0](https://github.com/epoberezkin/fast-json-stable-stringify) - MIT
[fast-levenshtein@2.0.6](https://github.com/hiddentao/fast-levenshtein) - MIT
[fast-xml-parser@4.2.5](https://github.com/NaturalIntelligence/fast-xml-parser) - MIT
[fastq@1.16.0](https://github.com/mcollina/fastq) - ISC
[fb-watchman@2.0.2](https://github.com/facebook/watchman) - Apache-2.0
[file-entry-cache@6.0.1](https://github.com/royriojas/file-entry-cache) - MIT
[fill-range@7.1.1](https://github.com/jonschlinkert/fill-range) - MIT
[find-replace@3.0.0](https://github.com/75lb/find-replace) - MIT
[find-replace@5.0.2](https://github.com/75lb/find-replace) - MIT
[find-up@4.1.0](https://github.com/sindresorhus/find-up) - MIT
[find-up@5.0.0](https://github.com/sindresorhus/find-up) - MIT
[flat-cache@3.2.0](https://github.com/jaredwray/flat-cache) - MIT
[flatbuffers@1.12.0](https://github.com/google/flatbuffers) - Apache*
[flatbuffers@23.5.26](https://github.com/google/flatbuffers) - Apache*
[flatbuffers@24.3.25](https://github.com/google/flatbuffers) - Apache-2.0
[flatbuffers@25.9.23](https://github.com/google/flatbuffers) - Apache-2.0
[flatted@3.2.9](https://github.com/WebReflection/flatted) - ISC
[follow-redirects@1.15.6](https://github.com/follow-redirects/follow-redirects) - MIT
[foreground-child@3.3.0](https://github.com/tapjs/foreground-child) - ISC
[form-data-encoder@1.7.2](https://github.com/octet-stream/form-data-encoder) - MIT
[form-data@4.0.0](https://github.com/form-data/form-data) - MIT
[formdata-node@4.4.1](https://github.com/octet-stream/form-data) - MIT
[fs.realpath@1.0.0](https://github.com/isaacs/fs.realpath) - ISC
[fsevents@2.3.3](https://github.com/fsevents/fsevents) - MIT
[function-bind@1.1.2](https://github.com/Raynos/function-bind) - MIT
[gensync@1.0.0-beta.2](https://github.com/loganfsmyth/gensync) - MIT
[get-caller-file@2.0.5](https://github.com/stefanpenner/get-caller-file) - ISC
[get-package-type@0.1.0](https://github.com/cfware/get-package-type) - MIT
[get-stream@6.0.1](https://github.com/sindresorhus/get-stream) - MIT
[glob-parent@5.1.2](https://github.com/gulpjs/glob-parent) - ISC
[glob-parent@6.0.2](https://github.com/gulpjs/glob-parent) - ISC
[glob@10.4.5](https://github.com/isaacs/node-glob) - ISC
[glob@7.2.3](https://github.com/isaacs/node-glob) - ISC
[globals@11.12.0](https://github.com/sindresorhus/globals) - MIT
[globals@13.24.0](https://github.com/sindresorhus/globals) - MIT
[globby@11.1.0](https://github.com/sindresorhus/globby) - MIT
[graceful-fs@4.2.11](https://github.com/isaacs/node-graceful-fs) - ISC
[graphemer@1.4.0](https://github.com/flmnt/graphemer) - MIT
[guid-typescript@1.0.9](https://github.com/NicolasDeveloper/guid-typescript) - ISC
[has-flag@4.0.0](https://github.com/sindresorhus/has-flag) - MIT
[hasown@2.0.0](https://github.com/inspect-js/hasOwn) - MIT
[html-escaper@2.0.2](https://github.com/WebReflection/html-escaper) - MIT
[human-signals@2.1.0](https://github.com/ehmicky/human-signals) - Apache-2.0
[humanize-ms@1.2.1](https://github.com/node-modules/humanize-ms) - MIT
[ignore@5.3.0](https://github.com/kaelzhang/node-ignore) - MIT
[import-fresh@3.3.0](https://github.com/sindresorhus/import-fresh) - MIT
[import-local@3.1.0](https://github.com/sindresorhus/import-local) - MIT
[imurmurhash@0.1.4](https://github.com/jensyt/imurmurhash-js) - MIT
[inflight@1.0.6](https://github.com/npm/inflight) - ISC
[inherits@2.0.4](https://github.com/isaacs/inherits) - ISC
[interpret@1.4.0](https://github.com/gulpjs/interpret) - MIT
[is-arrayish@0.2.1](https://github.com/qix-/node-is-arrayish) - MIT
[is-arrayish@0.3.2](https://github.com/qix-/node-is-arrayish) - MIT
[is-buffer@1.1.6](https://github.com/feross/is-buffer) - MIT
[is-core-module@2.13.1](https://github.com/inspect-js/is-core-module) - MIT
[is-extglob@2.1.1](https://github.com/jonschlinkert/is-extglob) - MIT
[is-fullwidth-code-point@3.0.0](https://github.com/sindresorhus/is-fullwidth-code-point) - MIT
[is-generator-fn@2.1.0](https://github.com/sindresorhus/is-generator-fn) - MIT
[is-glob@4.0.3](https://github.com/micromatch/is-glob) - MIT
[is-number@7.0.0](https://github.com/jonschlinkert/is-number) - MIT
[is-path-inside@3.0.3](https://github.com/sindresorhus/is-path-inside) - MIT
[is-stream@2.0.1](https://github.com/sindresorhus/is-stream) - MIT
[isexe@2.0.0](https://github.com/isaacs/isexe) - ISC
[istanbul-lib-coverage@3.2.2](https://github.com/istanbuljs/istanbuljs) - BSD-3-Clause
[istanbul-lib-instrument@5.2.1](https://github.com/istanbuljs/istanbuljs) - BSD-3-Clause
[istanbul-lib-instrument@6.0.1](https://github.com/istanbuljs/istanbuljs) - BSD-3-Clause
[istanbul-lib-report@3.0.1](https://github.com/istanbuljs/istanbuljs) - BSD-3-Clause
[istanbul-lib-source-maps@4.0.1](https://github.com/istanbuljs/istanbuljs) - BSD-3-Clause
[istanbul-reports@3.1.6](https://github.com/istanbuljs/istanbuljs) - BSD-3-Clause
[jackspeak@3.4.3](https://github.com/isaacs/jackspeak) - BlueOak-1.0.0
[jest-changed-files@29.7.0](https://github.com/jestjs/jest) - MIT
[jest-circus@29.7.0](https://github.com/jestjs/jest) - MIT
[jest-cli@29.7.0](https://github.com/jestjs/jest) - MIT
[jest-config@29.7.0](https://github.com/jestjs/jest) - MIT
[jest-diff@29.7.0](https://github.com/jestjs/jest) - MIT
[jest-docblock@29.7.0](https://github.com/jestjs/jest) - MIT
[jest-each@29.7.0](https://github.com/jestjs/jest) - MIT
[jest-environment-node@29.7.0](https://github.com/jestjs/jest) - MIT
[jest-get-type@29.6.3](https://github.com/jestjs/jest) - MIT
[jest-haste-map@29.7.0](https://github.com/jestjs/jest) - MIT
[jest-leak-detector@29.7.0](https://github.com/jestjs/jest) - MIT
[jest-matcher-utils@29.7.0](https://github.com/jestjs/jest) - MIT
[jest-message-util@29.7.0](https://github.com/jestjs/jest) - MIT
[jest-mock@29.7.0](https://github.com/jestjs/jest) - MIT
[jest-pnp-resolver@1.2.3](https://github.com/arcanis/jest-pnp-resolver) - MIT
[jest-regex-util@29.6.3](https://github.com/jestjs/jest) - MIT
[jest-resolve-dependencies@29.7.0](https://github.com/jestjs/jest) - MIT
[jest-resolve@29.7.0](https://github.com/jestjs/jest) - MIT
[jest-runner@29.7.0](https://github.com/jestjs/jest) - MIT
[jest-runtime@29.7.0](https://github.com/jestjs/jest) - MIT
[jest-snapshot@29.7.0](https://github.com/jestjs/jest) - MIT
[jest-util@29.7.0](https://github.com/jestjs/jest) - MIT
[jest-validate@29.7.0](https://github.com/jestjs/jest) - MIT
[jest-watcher@29.7.0](https://github.com/jestjs/jest) - MIT
[jest-worker@29.7.0](https://github.com/jestjs/jest) - MIT
[jest@29.7.0](https://github.com/jestjs/jest) - MIT
[js-tokens@4.0.0](https://github.com/lydell/js-tokens) - MIT
[js-yaml@3.14.1](https://github.com/nodeca/js-yaml) - MIT
[js-yaml@4.1.0](https://github.com/nodeca/js-yaml) - MIT
[jsesc@2.5.2](https://github.com/mathiasbynens/jsesc) - MIT
[json-bignum@0.0.3](https://github.com/datalanche/json-bignum) - MIT
[json-buffer@3.0.1](https://github.com/dominictarr/json-buffer) - MIT
[json-parse-even-better-errors@2.3.1](https://github.com/npm/json-parse-even-better-errors) - MIT
[json-schema-traverse@0.4.1](https://github.com/epoberezkin/json-schema-traverse) - MIT
[json-stable-stringify-without-jsonify@1.0.1](https://github.com/samn/json-stable-stringify) - MIT
[json5@2.2.3](https://github.com/json5/json5) - MIT
[keyv@4.5.4](https://github.com/jaredwray/keyv) - MIT
[kleur@3.0.3](https://github.com/lukeed/kleur) - MIT
[leven@3.1.0](https://github.com/sindresorhus/leven) - MIT
[levn@0.4.1](https://github.com/gkz/levn) - MIT
[lines-and-columns@1.2.4](https://github.com/eventualbuddha/lines-and-columns) - MIT
[linkify-it@5.0.0](https://github.com/markdown-it/linkify-it) - MIT
[locate-path@5.0.0](https://github.com/sindresorhus/locate-path) - MIT
[locate-path@6.0.0](https://github.com/sindresorhus/locate-path) - MIT
[lodash.camelcase@4.3.0](https://github.com/lodash/lodash) - MIT
[lodash.memoize@4.1.2](https://github.com/lodash/lodash) - MIT
[lodash.merge@4.6.2](https://github.com/lodash/lodash) - MIT
[lodash@4.17.21](https://github.com/lodash/lodash) - MIT
[long@5.2.3](https://github.com/dcodeIO/long.js) - Apache-2.0
[lru-cache@10.4.3](https://github.com/isaacs/node-lru-cache) - ISC
[lru-cache@5.1.1](https://github.com/isaacs/node-lru-cache) - ISC
[lunr@2.3.9](https://github.com/olivernn/lunr.js) - MIT
[make-dir@4.0.0](https://github.com/sindresorhus/make-dir) - MIT
[make-error@1.3.6](https://github.com/JsCommunity/make-error) - ISC
[makeerror@1.0.12](https://github.com/daaku/nodejs-makeerror) - BSD-3-Clause
[markdown-it@14.1.0](https://github.com/markdown-it/markdown-it) - MIT
[md5@2.3.0](https://github.com/pvorb/node-md5) - BSD-3-Clause
[mdurl@2.0.0](https://github.com/markdown-it/mdurl) - MIT
[merge-stream@2.0.0](https://github.com/grncdr/merge-stream) - MIT
[merge2@1.4.1](https://github.com/teambition/merge2) - MIT
[micromatch@4.0.8](https://github.com/micromatch/micromatch) - MIT
[mime-db@1.52.0](https://github.com/jshttp/mime-db) - MIT
[mime-types@2.1.35](https://github.com/jshttp/mime-types) - MIT
[mimic-fn@2.1.0](https://github.com/sindresorhus/mimic-fn) - MIT
[minimatch@3.1.2](https://github.com/isaacs/minimatch) - ISC
[minimatch@9.0.3](https://github.com/isaacs/minimatch) - ISC
[minimatch@9.0.5](https://github.com/isaacs/minimatch) - ISC
[minimist@1.2.8](https://github.com/minimistjs/minimist) - MIT
[minipass@7.1.2](https://github.com/isaacs/minipass) - ISC
[minizlib@3.0.1](https://github.com/isaacs/minizlib) - MIT
[mkdirp@3.0.1](https://github.com/isaacs/node-mkdirp) - MIT
[mnemonist@0.38.3](https://github.com/yomguithereal/mnemonist) - MIT
[ms@2.1.2](https://github.com/zeit/ms) - MIT
[ms@2.1.3](https://github.com/vercel/ms) - MIT
[natural-compare@1.4.0](https://github.com/litejs/natural-compare-lite) - MIT
[node-domexception@1.0.0](https://github.com/jimmywarting/node-domexception) - MIT
[node-fetch@2.7.0](https://github.com/bitinn/node-fetch) - MIT
[node-int64@0.4.0](https://github.com/broofa/node-int64) - MIT
[node-releases@2.0.14](https://github.com/chicoxyzzy/node-releases) - MIT
[normalize-path@3.0.0](https://github.com/jonschlinkert/normalize-path) - MIT
[npm-run-path@4.0.1](https://github.com/sindresorhus/npm-run-path) - MIT
[obliterator@1.6.1](https://github.com/yomguithereal/obliterator) - MIT
[once@1.4.0](https://github.com/isaacs/once) - ISC
[onetime@5.1.2](https://github.com/sindresorhus/onetime) - MIT
[onnxruntime-common@1.19.2](https://github.com/Microsoft/onnxruntime) - MIT
[onnxruntime-common@1.20.0-dev.20241016-2b8fc5529b](https://github.com/Microsoft/onnxruntime) - MIT
[onnxruntime-node@1.19.2](https://github.com/Microsoft/onnxruntime) - MIT
[onnxruntime-web@1.21.0-dev.20241024-d9ca84ef96](https://github.com/Microsoft/onnxruntime) - MIT
[openai@4.29.2](https://github.com/openai/openai-node) - Apache-2.0
[optionator@0.9.3](https://github.com/gkz/optionator) - MIT
[p-limit@2.3.0](https://github.com/sindresorhus/p-limit) - MIT
[p-limit@3.1.0](https://github.com/sindresorhus/p-limit) - MIT
[p-locate@4.1.0](https://github.com/sindresorhus/p-locate) - MIT
[p-locate@5.0.0](https://github.com/sindresorhus/p-locate) - MIT
[p-try@2.2.0](https://github.com/sindresorhus/p-try) - MIT
[package-json-from-dist@1.0.1](https://github.com/isaacs/package-json-from-dist) - BlueOak-1.0.0
[parent-module@1.0.1](https://github.com/sindresorhus/parent-module) - MIT
[parse-json@5.2.0](https://github.com/sindresorhus/parse-json) - MIT
[path-exists@4.0.0](https://github.com/sindresorhus/path-exists) - MIT
[path-is-absolute@1.0.1](https://github.com/sindresorhus/path-is-absolute) - MIT
[path-key@3.1.1](https://github.com/sindresorhus/path-key) - MIT
[path-parse@1.0.7](https://github.com/jbgutierrez/path-parse) - MIT
[path-scurry@1.11.1](https://github.com/isaacs/path-scurry) - BlueOak-1.0.0
[path-type@4.0.0](https://github.com/sindresorhus/path-type) - MIT
[picocolors@1.0.0](https://github.com/alexeyraspopov/picocolors) - ISC
[picomatch@2.3.1](https://github.com/micromatch/picomatch) - MIT
[pirates@4.0.6](https://github.com/danez/pirates) - MIT
[pkg-dir@4.2.0](https://github.com/sindresorhus/pkg-dir) - MIT
[platform@1.3.6](https://github.com/bestiejs/platform.js) - MIT
[prelude-ls@1.2.1](https://github.com/gkz/prelude-ls) - MIT
[pretty-format@29.7.0](https://github.com/jestjs/jest) - MIT
[prompts@2.4.2](https://github.com/terkelg/prompts) - MIT
[protobufjs@7.4.0](https://github.com/protobufjs/protobuf.js) - BSD-3-Clause
[proxy-from-env@1.1.0](https://github.com/Rob--W/proxy-from-env) - MIT
[punycode.js@2.3.1](https://github.com/mathiasbynens/punycode.js) - MIT
[punycode@2.3.1](https://github.com/mathiasbynens/punycode.js) - MIT
[pure-rand@6.0.4](https://github.com/dubzzz/pure-rand) - MIT
[queue-microtask@1.2.3](https://github.com/feross/queue-microtask) - MIT
[react-is@18.2.0](https://github.com/facebook/react) - MIT
[rechoir@0.6.2](https://github.com/tkellen/node-rechoir) - MIT
[reflect-metadata@0.2.2](https://github.com/rbuckton/reflect-metadata) - Apache-2.0
[require-directory@2.1.1](https://github.com/troygoode/node-require-directory) - MIT
[resolve-cwd@3.0.0](https://github.com/sindresorhus/resolve-cwd) - MIT
[resolve-from@4.0.0](https://github.com/sindresorhus/resolve-from) - MIT
[resolve-from@5.0.0](https://github.com/sindresorhus/resolve-from) - MIT
[resolve.exports@2.0.2](https://github.com/lukeed/resolve.exports) - MIT
[resolve@1.22.8](https://github.com/browserify/resolve) - MIT
[reusify@1.0.4](https://github.com/mcollina/reusify) - MIT
[rimraf@3.0.2](https://github.com/isaacs/rimraf) - ISC
[rimraf@5.0.10](https://github.com/isaacs/rimraf) - ISC
[run-parallel@1.2.0](https://github.com/feross/run-parallel) - MIT
[semver@6.3.1](https://github.com/npm/node-semver) - ISC
[semver@7.6.3](https://github.com/npm/node-semver) - ISC
[sharp@0.33.5](https://github.com/lovell/sharp) - Apache-2.0
[shebang-command@2.0.0](https://github.com/kevva/shebang-command) - MIT
[shebang-regex@3.0.0](https://github.com/sindresorhus/shebang-regex) - MIT
[shelljs@0.8.5](https://github.com/shelljs/shelljs) - BSD-3-Clause
[shiki@1.10.3](https://github.com/shikijs/shiki) - MIT
[shx@0.3.4](https://github.com/shelljs/shx) - MIT
[signal-exit@3.0.7](https://github.com/tapjs/signal-exit) - ISC
[signal-exit@4.1.0](https://github.com/tapjs/signal-exit) - ISC
[simple-swizzle@0.2.2](https://github.com/qix-/node-simple-swizzle) - MIT
[sisteransi@1.0.5](https://github.com/terkelg/sisteransi) - MIT
[slash@3.0.0](https://github.com/sindresorhus/slash) - MIT
[source-map-support@0.5.13](https://github.com/evanw/node-source-map-support) - MIT
[source-map@0.6.1](https://github.com/mozilla/source-map) - BSD-3-Clause
[sprintf-js@1.0.3](https://github.com/alexei/sprintf.js) - BSD-3-Clause
[stack-utils@2.0.6](https://github.com/tapjs/stack-utils) - MIT
[stream-read-all@3.0.1](https://github.com/75lb/stream-read-all) - MIT
[string-length@4.0.2](https://github.com/sindresorhus/string-length) - MIT
[string-width@4.2.3](https://github.com/sindresorhus/string-width) - MIT
[string-width@5.1.2](https://github.com/sindresorhus/string-width) - MIT
[strip-ansi@6.0.1](https://github.com/chalk/strip-ansi) - MIT
[strip-ansi@7.1.0](https://github.com/chalk/strip-ansi) - MIT
[strip-bom@4.0.0](https://github.com/sindresorhus/strip-bom) - MIT
[strip-final-newline@2.0.0](https://github.com/sindresorhus/strip-final-newline) - MIT
[strip-json-comments@3.1.1](https://github.com/sindresorhus/strip-json-comments) - MIT
[strnum@1.0.5](https://github.com/NaturalIntelligence/strnum) - MIT
[supports-color@7.2.0](https://github.com/chalk/supports-color) - MIT
[supports-color@8.1.1](https://github.com/chalk/supports-color) - MIT
[supports-preserve-symlinks-flag@1.0.0](https://github.com/inspect-js/node-supports-preserve-symlinks-flag) - MIT
[table-layout@3.0.2](https://github.com/75lb/table-layout) - MIT
[tar@7.4.3](https://github.com/isaacs/node-tar) - ISC
[test-exclude@6.0.0](https://github.com/istanbuljs/test-exclude) - ISC
[text-table@0.2.0](https://github.com/substack/text-table) - MIT
[tmp@0.2.3](https://github.com/raszi/node-tmp) - MIT
[tmpl@1.0.5](https://github.com/daaku/nodejs-tmpl) - BSD-3-Clause
[to-regex-range@5.0.1](https://github.com/micromatch/to-regex-range) - MIT
[tr46@0.0.3](https://github.com/Sebmaster/tr46.js) - MIT
[ts-api-utils@1.0.3](https://github.com/JoshuaKGoldberg/ts-api-utils) - MIT
[ts-jest@29.1.2](https://github.com/kulshekhar/ts-jest) - MIT
[tslib@1.14.1](https://github.com/Microsoft/tslib) - 0BSD
[tslib@2.6.2](https://github.com/Microsoft/tslib) - 0BSD
[type-check@0.4.0](https://github.com/gkz/type-check) - MIT
[type-detect@4.0.8](https://github.com/chaijs/type-detect) - MIT
[type-fest@0.20.2](https://github.com/sindresorhus/type-fest) - (MIT OR CC0-1.0)
[type-fest@0.21.3](https://github.com/sindresorhus/type-fest) - (MIT OR CC0-1.0)
[typedoc-plugin-markdown@4.2.1](https://github.com/typedoc2md/typedoc-plugin-markdown) - MIT
[typedoc@0.26.4](https://github.com/TypeStrong/TypeDoc) - Apache-2.0
[typescript-eslint@7.1.0](https://github.com/typescript-eslint/typescript-eslint) - MIT
[typescript@5.5.4](https://github.com/Microsoft/TypeScript) - Apache-2.0
[typical@4.0.0](https://github.com/75lb/typical) - MIT
[typical@7.1.1](https://github.com/75lb/typical) - MIT
[uc.micro@2.1.0](https://github.com/markdown-it/uc.micro) - MIT
[undici-types@5.26.5](https://github.com/nodejs/undici) - MIT
[undici-types@6.19.8](https://github.com/nodejs/undici) - MIT
[undici-types@7.24.6](https://github.com/nodejs/undici) - MIT
[update-browserslist-db@1.0.13](https://github.com/browserslist/update-db) - MIT
[uri-js@4.4.1](https://github.com/garycourt/uri-js) - BSD-2-Clause
[uuid@9.0.1](https://github.com/uuidjs/uuid) - MIT
[v8-to-istanbul@9.2.0](https://github.com/istanbuljs/v8-to-istanbul) - ISC
[walker@1.0.8](https://github.com/daaku/nodejs-walker) - Apache-2.0
[web-streams-polyfill@3.3.3](https://github.com/MattiasBuelens/web-streams-polyfill) - MIT
[web-streams-polyfill@4.0.0-beta.3](https://github.com/MattiasBuelens/web-streams-polyfill) - MIT
[webidl-conversions@3.0.1](https://github.com/jsdom/webidl-conversions) - BSD-2-Clause
[whatwg-url@5.0.0](https://github.com/jsdom/whatwg-url) - MIT
[which@2.0.2](https://github.com/isaacs/node-which) - ISC
[wordwrapjs@5.1.0](https://github.com/75lb/wordwrapjs) - MIT
[wrap-ansi@7.0.0](https://github.com/chalk/wrap-ansi) - MIT
[wrap-ansi@8.1.0](https://github.com/chalk/wrap-ansi) - MIT
[wrappy@1.0.2](https://github.com/npm/wrappy) - ISC
[write-file-atomic@4.0.2](https://github.com/npm/write-file-atomic) - ISC
[y18n@5.0.8](https://github.com/yargs/y18n) - ISC
[yallist@3.1.1](https://github.com/isaacs/yallist) - ISC
[yallist@5.0.0](https://github.com/isaacs/yallist) - BlueOak-1.0.0
[yaml@2.4.5](https://github.com/eemeli/yaml) - ISC
[yargs-parser@21.1.1](https://github.com/yargs/yargs-parser) - ISC
[yargs@17.7.2](https://github.com/yargs/yargs) - MIT
[yocto-queue@0.1.0](https://github.com/sindresorhus/yocto-queue) - MIT
nodejs/README.md
+1 -1
View File
@@ -30,7 +30,7 @@ const results = await table.vectorSearch([0.1, 0.3]).limit(20).toArray();
console.log(results);
```
The [quickstart](https://lancedb.github.io/lancedb/basic/) contains a more complete example.
The [quickstart](https://docs.lancedb.com/quickstart/) contains more complete examples.
## Development
nodejs/RUST_THIRD_PARTY_LICENSES.html
+14607
View File
File diff suppressed because it is too large Load Diff
nodejs/__test__/arrow.test.ts
+1545 -911
View File
File diff suppressed because it is too large Load Diff
nodejs/__test__/arrow_versions.ts
+39
View File
@@ -0,0 +1,39 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// Every Apache Arrow release the package accepts as a peer dependency, as
// separately installed copies so the tests can feed objects built by one
// Arrow version into a LanceDB running on another. When a new Arrow major
// is released, add the alias to `package.json` and extend this list; the
// test files pick the change up from here.
import * as arrow15 from "apache-arrow-15";
import * as arrow16 from "apache-arrow-16";
import * as arrow17 from "apache-arrow-17";
import * as arrow18 from "apache-arrow-18";
import * as arrow19 from "apache-arrow-19";
import * as arrow20 from "apache-arrow-20";
import * as arrow21 from "apache-arrow-21";
export { arrow15, arrow16, arrow17, arrow18, arrow19, arrow20, arrow21 };
export type ApacheArrow =
| typeof arrow15
| typeof arrow16
| typeof arrow17
| typeof arrow18
| typeof arrow19
| typeof arrow20
| typeof arrow21;
export const arrowVersions: ApacheArrow[] = [
arrow15,
arrow16,
arrow17,
arrow18,
arrow19,
arrow20,
arrow21,
];
/** The newest supported Arrow release, for tests that only need one. */
export const latestArrow = arrow21;
nodejs/__test__/blob.test.ts
+695
View File
@@ -0,0 +1,695 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import {
Field,
Float,
Float32,
Int32,
Int64,
List,
Schema,
Struct,
Utf8,
} from "apache-arrow";
import {
convertToTable,
makeArrowTable,
resolveBlobInputs,
} from "../lancedb/arrow";
import { BlobFile, blob, coerceBlobValue, isBlobField } from "../lancedb/blob";
import { EmbeddingFunction, LanceSchema } from "../lancedb/embedding";
import { getRegistry } from "../lancedb/embedding/registry";
function legacyBlob(name: string): Field {
const field = blob(name);
return new Field(
name,
new Struct((field.type as Struct).children.slice(0, 2)),
field.nullable,
field.metadata,
);
}
describe("blob()", () => {
it("declares the four-field range layout", () => {
const field = blob("image", {
nullable: false,
inlineSizeThreshold: 12,
});
expect(field.nullable).toBe(false);
expect(isBlobField(field)).toBe(true);
expect(
field.metadata.get("lance-encoding:blob-inline-size-threshold"),
).toBe("12");
expect(
(field.type as Struct).children.map((f) => [
f.name,
f.type.toString(),
f.nullable,
]),
).toEqual([
["data", "LargeBinary", true],
["uri", "Utf8", true],
["position", "Uint64", true],
["size", "Uint64", true],
]);
});
it("marks the field as lance.blob.v2", () => {
const field = blob("image", { nullable: false });
expect(field.nullable).toBe(false);
expect(isBlobField(field)).toBe(true);
expect(field.metadata.get("ARROW:extension:name")).toBe("lance.blob.v2");
});
it("writes encoding thresholds as field metadata", () => {
const field = blob("video", {
inlineSizeThreshold: 1024,
dedicatedSizeThreshold: 2 * 1024 * 1024,
packFileSizeThreshold: 64 * 1024 * 1024,
});
expect(
field.metadata.get("lance-encoding:blob-inline-size-threshold"),
).toBe("1024");
expect(
field.metadata.get("lance-encoding:blob-dedicated-size-threshold"),
).toBe(String(2 * 1024 * 1024));
expect(
field.metadata.get("lance-encoding:blob-pack-file-size-threshold"),
).toBe(String(64 * 1024 * 1024));
});
it("rejects invalid thresholds", () => {
expect(() => blob("image", { inlineSizeThreshold: -1 })).toThrow(
/inlineSizeThreshold must be non-negative/,
);
expect(() => blob("image", { dedicatedSizeThreshold: 0 })).toThrow(
/dedicatedSizeThreshold must be positive/,
);
expect(() => blob("image", { packFileSizeThreshold: 1.5 })).toThrow(
/packFileSizeThreshold must be a safe integer/,
);
expect(() =>
blob("image", { dedicatedSizeThreshold: Number.MAX_SAFE_INTEGER + 1 }),
).toThrow(/dedicatedSizeThreshold must be a safe integer/);
});
});
class BlobStruct {
constructor(
public data?: unknown,
public uri?: unknown,
) {}
}
describe("coerceBlobValue", () => {
it.each([
["Buffer", Buffer.from("x"), { data: Buffer.from("x"), uri: null }],
[
"Uint8Array",
new Uint8Array([120]),
{ data: new Uint8Array([120]), uri: null },
],
["URI string", "s3://bucket/key", { data: null, uri: "s3://bucket/key" }],
[
"data struct",
{ data: Buffer.from("y") },
{ data: Buffer.from("y"), uri: null },
],
[
"uri struct",
{ uri: "s3://bucket/key" },
{ data: null, uri: "s3://bucket/key" },
],
[
"ArrayBuffer",
new Uint8Array([120]).buffer,
{ data: new Uint8Array([120]), uri: null },
],
[
"URL",
new URL("https://example.com/blob"),
{ data: null, uri: "https://example.com/blob" },
],
[
"data struct with ArrayBuffer",
{ data: new Uint8Array([121]).buffer },
{ data: new Uint8Array([121]), uri: null },
],
[
"uri struct with URL",
{ uri: new URL("file:///tmp/a.png") },
{ data: null, uri: "file:///tmp/a.png" },
],
[
"class instance with Uint8Array data",
new BlobStruct(new Uint8Array([122])),
{ data: new Uint8Array([122]), uri: null },
],
[
"class instance with ArrayBuffer data",
new BlobStruct(new Uint8Array([122]).buffer),
{ data: new Uint8Array([122]), uri: null },
],
[
"class instance with URL uri",
new BlobStruct(undefined, new URL("s3://bucket/key")),
{ data: null, uri: "s3://bucket/key" },
],
["null", null, null],
])("accepts %s", (_name, input, expected) => {
expect(coerceBlobValue(input)).toEqual(expected);
});
it.each([
["empty URI", "", /uri cannot be empty/],
["object without data or uri", { position: 0 }, /data' or 'uri/],
[
"Int16Array",
new Int16Array([1]),
/Blob data must be Buffer, Uint8Array, ArrayBuffer, or Blob/,
],
[
"both data and uri",
{ data: Buffer.from("y"), uri: "s3://bucket/key" },
/exactly one of 'data' or 'uri'/,
],
[
"neither data nor uri",
{ data: null, uri: null },
/exactly one of 'data' or 'uri'/,
],
["Blob", new Blob(["x"]), /must be read asynchronously/],
["data struct with Blob", { data: new Blob(["x"]) }, /asynchronously/],
[
"class instance with Blob data",
new BlobStruct(new Blob(["x"])),
/asynchronously/,
],
[
"unknown struct field",
{ data: Buffer.from("y"), mime: "image/png" },
/Blob struct values only support .*got 'mime'/,
],
[
"misspelled struct field",
{ uri: "s3://bucket/key", dat: Buffer.from("y") },
/got 'dat'/,
],
[
"SharedArrayBuffer",
new SharedArrayBuffer(2),
/Blob column values must be Buffer, Uint8Array, ArrayBuffer, Blob/,
],
["Date", new Date(0), /Blob column values must be/],
])("rejects %s", (_name, input, message) => {
expect(() => coerceBlobValue(input)).toThrow(message);
});
});
describe("external blob ranges", () => {
const uri = "s3://bucket/key";
it.each([0, 4, 0n, 4n, 9007199254740993n])(
"preserves position %s",
(position) => {
expect(coerceBlobValue({ uri, position, size: 6 })).toEqual({
data: null,
uri,
position: BigInt(position),
size: 6n,
});
},
);
it.each([
[{ uri, position: 0 }, /both position and size/],
[{ uri, size: 6 }, /both position and size/],
[{ uri, position: null, size: 6 }, /both position and size/],
[{ uri, position: 0, size: 0 }, /size must be positive/],
[{ uri, position: -1, size: 6 }, /position must be non-negative/],
[{ uri, position: 0, size: -1n }, /size must be positive/],
[{ uri, position: 0.5, size: 6 }, /safe integer or bigint/],
[
{ uri, position: Number.MAX_SAFE_INTEGER + 1, size: 6 },
/safe integer or bigint/,
],
[{ uri, position: 0, size: Number.NaN }, /safe integer or bigint/],
[{ uri, position: "4", size: 6 }, /safe integer or bigint/],
[{ uri, position: false, size: 6 }, /safe integer or bigint/],
[{ uri, position: 1n << 64n, size: 6 }, /fit in uint64/],
[{ uri, position: 0xffffffffffffffffn, size: 1 }, /overflows uint64/],
[{ data: Buffer.from("x"), position: 0, size: 6 }, /require a uri/],
])("rejects malformed input %p", (value, message) => {
expect(() => coerceBlobValue(value)).toThrow(message);
});
it("preserves ranges while normalizing a URL held by getters", () => {
class Range {
get uri() {
return new URL(uri);
}
get position() {
return 4;
}
get size() {
return 6;
}
}
expect(coerceBlobValue(new Range())).toEqual({
data: null,
uri,
position: 4n,
size: 6n,
});
});
it("writes ranges and mixed whole-file, inline, and null values to Arrow", () => {
const schema = new Schema([blob("image")]);
const rows = [
null,
Buffer.from("x"),
uri,
{ uri, position: 4, size: 6 },
].map((image) => ({ image }));
const column = makeArrowTable(rows, { schema }).getChild("image")!;
expect(column.get(0)).toBeNull();
expect(Buffer.from(column.getChild("data")!.get(1))).toEqual(
Buffer.from("x"),
);
expect(column.getChild("uri")!.get(2)).toBe(uri);
expect(column.getChild("position")!.toArray()).toEqual(
new BigUint64Array([0n, 0n, 0n, 4n]),
);
expect(column.getChild("position")!.get(2)).toBeNull();
expect(column.getChild("size")!.get(3)).toBe(6n);
});
it.each([
Buffer.from("inline"),
uri,
{ uri, position: null, size: null },
null,
])("retains the full declared range layout when seeded with %p", (image) => {
const schema = new Schema([blob("image")]);
const table = makeArrowTable([{ image }], { schema });
expect(table.schema).toEqual(schema);
expect(table.getChild("image")!.getChild("position")!.get(0)).toBeNull();
});
it.each([0, 4])(
"rejects position %s on a minimal schema before dropping keys",
(position) => {
expect(() =>
makeArrowTable(
[
{ image: Buffer.from("inline") },
{ image: { uri, position, size: 6 } },
],
{ schema: new Schema([legacyBlob("image")]) },
),
).toThrow(
/field image at row 1: Blob storage schema cannot store byte ranges/,
);
},
);
it("accepts omitted or null range fields on the minimal schema", () => {
const column = makeArrowTable(
[{ image: { uri, position: null, size: null } }],
{
schema: new Schema([legacyBlob("image")]),
},
).getChild("image")!;
expect(column.getChild("uri")!.get(0)).toBe(uri);
});
it("validates ranges nested in structs and lists against their storage fields", () => {
const value = { uri, position: 4, size: 6 };
for (const field of [legacyBlob("image"), blob("image")]) {
const schema = new Schema([
new Field("info", new Struct([field]), true),
new Field("images", new List(field), true),
]);
if ((field.type as Struct).children.length === 2) {
expect(() =>
makeArrowTable([{ info: { image: value } }], { schema }),
).toThrow(/field info.image.*storage schema/);
expect(() => makeArrowTable([{ images: [value] }], { schema })).toThrow(
/field images\[0\].*storage schema/,
);
} else {
const table = makeArrowTable(
[{ info: { image: value }, images: [value] }],
{ schema },
);
const info = table.getChild("info")!.get(0) as {
image: { position: bigint; size: bigint };
};
expect(info.image.position).toBe(4n);
const images = Array.from(
table.getChild("images")!.get(0) as Iterable<{
position: bigint;
size: bigint;
}>,
);
expect(images[0].size).toBe(6n);
}
}
});
});
describe("resolveBlobInputs", () => {
it("reads Blob and File values in blob columns only", async () => {
const schema = new Schema([
new Field("name", new Utf8(), true),
blob("image"),
new Field("images", new List(blob("image")), true),
]);
const shared = new Blob(["same"]);
const notABlobColumn = new Blob(["other"]);
const data = [
{
name: notABlobColumn,
image: new File(["file"], "a.txt"),
images: [shared, { data: shared }, null],
},
{ name: "x", image: Buffer.from("buf"), images: null },
];
const resolved = await resolveBlobInputs(data, schema);
expect(resolved[0].name).toBe(notABlobColumn);
expect(Buffer.from(resolved[0].image as Uint8Array).toString()).toBe(
"file",
);
const images = resolved[0].images as [Uint8Array, { data: Uint8Array }];
expect(Buffer.from(images[0]).toString()).toBe("same");
expect(Buffer.from(images[1].data).toString()).toBe("same");
// Rows without Blob values and the caller's records are left alone.
expect(resolved[1]).toBe(data[1]);
expect(data[0].images?.[0]).toBe(shared);
});
it("reads each distinct Blob once, bare or inside data structs", async () => {
class CountingBlob extends Blob {
calls = 0;
async arrayBuffer(): Promise<ArrayBuffer> {
this.calls++;
return super.arrayBuffer();
}
}
const shared = new CountingBlob(["same"]);
const schema = new Schema([blob("image")]);
const resolved = await resolveBlobInputs(
[
{ image: { data: shared } },
{ image: { data: shared } },
{ image: shared },
],
schema,
);
expect(shared.calls).toBe(1);
const first = (resolved[0].image as { data: Uint8Array }).data;
expect(Buffer.from(first).toString()).toBe("same");
expect((resolved[1].image as { data: Uint8Array }).data).toBe(first);
expect(resolved[2].image).toBe(first);
});
it("reads a Blob held by a class instance", async () => {
const schema = new Schema([blob("image")]);
const [row] = await resolveBlobInputs(
[{ image: new BlobStruct(new Blob(["hi"])) }],
schema,
);
expect(
Buffer.from((row.image as { data: Uint8Array }).data).toString(),
).toBe("hi");
});
it("returns the input when there is nothing to read", async () => {
const data = [{ image: Buffer.from("x") }];
const schema = new Schema([blob("image")]);
await expect(resolveBlobInputs(data, schema)).resolves.toBe(data);
await expect(resolveBlobInputs(data)).resolves.toBe(data);
});
});
describe("convertToTable blob columns with embedding functions", () => {
// Embedding metadata routes rows through ensureNestedFieldsExist, which must
// leave blob values for the blob coercion instead of treating them as structs.
class LengthEmbedding extends EmbeddingFunction<string> {
ndims() {
return 2;
}
embeddingDataType(): Float {
return new Float32();
}
async computeSourceEmbeddings(data: string[]) {
return data.map((text) => [text.length, 1]);
}
}
getRegistry().register("blob-test-length")(LengthEmbedding);
const func = getRegistry().get<LengthEmbedding>("blob-test-length")!.create();
const embeddingSchema = LanceSchema({
text: func.sourceField(new Utf8()),
vector: func.vectorField(),
});
const schema = new Schema(
[...embeddingSchema.fields, blob("image")],
embeddingSchema.metadata,
);
it.each([
["Uint8Array", new Uint8Array([104, 105]), "hi", null],
["ArrayBuffer", new Uint8Array([104, 105]).buffer, "hi", null],
["Blob", new Blob(["hi"]), "hi", null],
["URI string", "s3://b/k", null, "s3://b/k"],
["URL", new URL("s3://b/k"), null, "s3://b/k"],
["data struct", { data: new Uint8Array([104, 105]) }, "hi", null],
])("accepts %s input", async (_name, image, data, uri) => {
const table = await convertToTable([{ text: "a", image }], undefined, {
schema,
});
const column = table.getChild("image")!;
expect(column.nullCount).toBe(0);
const bytes = column.getChild("data")!.get(0);
expect(bytes == null ? null : Buffer.from(bytes).toString()).toBe(data);
expect(column.getChild("uri")!.get(0)).toBe(uri);
});
it("writes a missing blob as null", async () => {
const table = await convertToTable([{ text: "a" }], undefined, { schema });
const column = table.getChild("image")!;
expect(column.nullCount).toBe(1);
expect(column.get(0)).toBeNull();
});
});
describe("BlobFile", () => {
it("rejects constructing BlobFile without a native handle", () => {
expect(() => new (BlobFile as unknown as { new (): BlobFile })()).toThrow(
/fetchBlobFiles/,
);
});
});
describe("makeArrowTable blob columns", () => {
it.each([
["ReadableStream", new ReadableStream()],
["Int16Array", new Int16Array([1])],
])("rejects %s input rather than writing a null blob", (_name, image) => {
const schema = new Schema([blob("image")]);
expect(() => makeArrowTable([{ image }], { schema })).toThrow(/image/);
});
it.each([
["Blob", new Blob(["hello"])],
["File", new File(["hello"], "hello.txt")],
["data struct with Blob", { data: new Blob(["hello"]) }],
[
"Blob with enumerable fields",
Object.assign(new Blob(["hello"]), { position: 0 }),
],
])("points %s input at the async APIs", (_name, image) => {
const schema = new Schema([blob("image")]);
expect(() => makeArrowTable([{ id: 1, image }], { schema })).toThrow(
/blob field image at row 0: Blob and File values must be read asynchronously/,
);
});
it.each([
["ArrayBuffer", new Uint8Array([104, 105]).buffer, "hi", null],
[
"data struct with ArrayBuffer",
{ data: new Uint8Array([104, 105]).buffer },
"hi",
null,
],
["URL", new URL("https://example.com/a"), null, "https://example.com/a"],
[
"uri struct with URL",
{ uri: new URL("file:///tmp/a.png") },
null,
"file:///tmp/a.png",
],
])("accepts %s input", (_name, image, data, uri) => {
const schema = new Schema([blob("image")]);
const input = { image };
const table = makeArrowTable([input], { schema });
const column = table.getChild("image")!;
expect(column.nullCount).toBe(0);
const bytes = column.getChild("data")!.get(0);
expect(bytes == null ? null : Buffer.from(bytes).toString()).toBe(data);
expect(column.getChild("uri")!.get(0)).toBe(uri);
// The caller's record is left untouched.
expect(input.image).toBe(image);
});
it.each([
["Uint8Array", new Uint8Array([104, 105]), "hi", null],
["URI string", "s3://b/k", null, "s3://b/k"],
["data struct", { data: new Uint8Array([104, 105]) }, "hi", null],
])("accepts %s input after a null row", (_name, image, data, uri) => {
const schema = new Schema([blob("image")]);
const table = makeArrowTable([{ image: null }, { image }], { schema });
const column = table.getChild("image")!;
expect(column.nullCount).toBe(1);
expect(column.get(0)).toBeNull();
const bytes = column.getChild("data")!.get(1);
expect(bytes == null ? null : Buffer.from(bytes).toString()).toBe(data);
expect(column.getChild("uri")!.get(1)).toBe(uri);
});
it("accepts a null struct before one holding a blob", () => {
const schema = new Schema([
new Field("info", new Struct([blob("image")]), true),
]);
const table = makeArrowTable(
[{ info: null }, { info: { image: new Uint8Array([97]) } }],
{ schema },
);
const info = table.getChild("info")!;
expect(info.get(0)).toBeNull();
const image = (info.get(1) as { image: { data: Uint8Array } }).image;
expect(Buffer.from(image.data).toString()).toBe("a");
});
it("skips a missing blob column named like an Object.prototype member", () => {
const schema = new Schema([
new Field("id", new Int32(), true),
blob("constructor"),
]);
const table = makeArrowTable([{ id: 1 }], { schema });
expect(table.schema.fields.map((f) => f.name)).toEqual(["id"]);
});
it("accepts ArrayBuffer and URL inside lists and structs", () => {
const schema = new Schema([
new Field("images", new List(blob("image")), true),
new Field("info", new Struct([blob("image")]), true),
]);
const table = makeArrowTable(
[
{
images: [new Uint8Array([97]).buffer, new URL("s3://b/k")],
info: { image: new Uint8Array([98]).buffer },
},
],
{ schema },
);
const images = Array.from(
table.getChild("images")!.get(0) as Iterable<{
data: Uint8Array | null;
uri: string | null;
}>,
);
expect(Buffer.from(images[0].data!).toString()).toBe("a");
expect(images[1].uri).toBe("s3://b/k");
const info = table.getChild("info")!.get(0) as {
image: { data: Uint8Array };
};
expect(Buffer.from(info.image.data).toString()).toBe("b");
});
it("coerces Buffer input onto a blob field", () => {
const schema = new Schema([
new Field("id", new Int64(), true),
blob("image"),
]);
const table = makeArrowTable([{ id: 1n, image: Buffer.from("hello") }], {
schema,
});
expect(isBlobField(table.schema.fields[1])).toBe(true);
const image = table.getChild("image")!;
expect(image.nullCount).toBe(0);
expect(image.getChild("uri")!.get(0)).toBeNull();
expect(image.getChild("data")!.nullCount).toBe(0);
expect(Buffer.from(image.getChild("data")!.get(0)!).toString()).toBe(
"hello",
);
});
it("coerces Buffer elements inside a list and keeps null slots", () => {
const schema = new Schema([
new Field("id", new Int64(), true),
new Field("images", new List(blob("image")), true),
]);
const table = makeArrowTable(
[
{ id: 1n, images: [Buffer.from("a"), Buffer.from("bb")] },
{ id: 2n, images: null },
{ id: 3n, images: [Buffer.from("c"), null] },
{ id: 4n, images: [] },
],
{ schema },
);
const images = table.getChild("images")!;
expect(images.nullCount).toBe(1);
const rows = images.toArray();
expect(rows[1]).toBeNull();
expect(Array.from(rows[3] as Iterable<unknown>)).toHaveLength(0);
const first = Array.from(rows[0] as Iterable<{ data: Uint8Array | null }>);
expect(Buffer.from(first[0].data!).toString()).toBe("a");
expect(Buffer.from(first[1].data!).toString()).toBe("bb");
const third = Array.from(
rows[2] as Iterable<{ data: Uint8Array | null } | null>,
);
expect(Buffer.from(third[0]!.data!).toString()).toBe("c");
expect(third[1]).toBeNull();
});
it("coerces Buffer fields inside list structs", () => {
const schema = new Schema([
new Field("id", new Int64(), true),
new Field(
"items",
new List(
new Field(
"item",
new Struct([new Field("name", new Utf8(), true), blob("image")]),
true,
),
),
true,
),
]);
const table = makeArrowTable(
[
{
id: 1n,
items: [{ name: "one", image: Buffer.from("alpha") }],
},
],
{ schema },
);
const items = Array.from(
table.getChild("items")!.toArray()[0] as Iterable<{
name: string;
image: { data: Uint8Array | null };
}>,
);
expect(items[0].name).toBe("one");
expect(Buffer.from(items[0].image.data!).toString()).toBe("alpha");
});
});
nodejs/__test__/catalog.test.ts
+306
View File
@@ -0,0 +1,306 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import * as http from "http";
import { Catalog, connectCatalog } from "../lancedb";
type RecordedRequest = {
url: string;
headers: http.IncomingHttpHeaders;
body: Record<string, unknown>;
};
async function withCatalog(
responses: [number, unknown][],
callback: (catalog: Catalog, requests: RecordedRequest[]) => Promise<void>,
) {
const requests: RecordedRequest[] = [];
const server = http.createServer(async (req, res) => {
const chunks: Buffer[] = [];
for await (const chunk of req) chunks.push(Buffer.from(chunk));
const body = Buffer.concat(chunks).toString();
requests.push({
url: req.url ?? "",
headers: req.headers,
body: body ? JSON.parse(body) : {},
});
const [status, response] = responses.shift() ?? [
500,
{ error: "Unexpected request" },
];
res.writeHead(status, { "content-type": "application/json" });
res.end(status === 204 ? undefined : JSON.stringify(response));
});
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
const address = server.address();
if (!address || typeof address === "string")
throw new Error("Missing server address");
try {
const catalog = await connectCatalog(`http://127.0.0.1:${address.port}`, {
apiKey: "secret",
sqlHostOverride: "grpc+tls://sql.example.com:10026",
clientConfig: {
extraHeaders: {
"X-LanceDB-Database": "wrong-static",
"X-LanceDB-Database-Prefix": "wrong",
},
},
headerProvider: () => ({
"X-LanceDB-Database": "wrong-dynamic",
"X-LanceDB-Database-Prefix": "wrong",
authorization: "Bearer refreshed",
}),
});
await callback(catalog, requests);
expect(responses).toHaveLength(0);
} finally {
server.closeAllConnections();
await new Promise<void>((resolve) => server.close(() => resolve()));
}
}
describe("remote catalog", () => {
it("uses root namespace routes and preserves independent database scope", async () => {
await withCatalog(
[
[204, null],
[200, { tables: [] }],
[200, {}],
[200, { tables: [] }],
[200, { tables: [] }],
// biome-ignore lint/style/useNamingConvention: server wire format
[200, { namespaces: ["team/search"], page_token: "next" }],
[204, null],
],
async (catalog, requests) => {
const first = await catalog.createDatabase("team/search", {
existOk: true,
});
expect(await first.tableNames()).toEqual([]);
const second = await catalog.connectDatabase("other");
expect(await second.tableNames()).toEqual([]);
expect(await first.tableNames()).toEqual([]);
const names = catalog.listDatabases({ pageLimit: 1, pageToken: "a/b" });
expect(await names.next()).toEqual({
done: false,
value: "team/search",
});
expect(names.pageToken()).toBe("next");
await catalog.dropDatabase("team/search", { ignoreMissing: true });
expect(requests[0].url).toBe("/v1/namespace/team%2Fsearch/create");
expect(requests[0].body).toEqual({ mode: "ExistOk" });
expect(requests[5].url).toBe(
"/v1/namespace/$/list?limit=1&page_token=a%2Fb",
);
expect(requests[6].body).toEqual({
mode: "Skip",
behavior: "Restrict",
});
for (const [i, request] of requests.entries()) {
expect(request.headers["x-lancedb-database"]).toBe(
i === 1 || i === 4 ? "team/search" : i === 3 ? "other" : undefined,
);
expect(request.headers["x-lancedb-database-prefix"]).toBeUndefined();
expect(request.headers.authorization).toBe("Bearer refreshed");
}
},
);
});
it("propagates lifecycle errors and sends restricted drops", async () => {
await withCatalog(
[
[404, {}],
[409, {}],
[400, {}],
[404, {}],
],
async (catalog, requests) => {
await expect(catalog.connectDatabase("missing")).rejects.toThrow(
"missing",
);
await expect(catalog.createDatabase("exists")).rejects.toThrow(
"exists",
);
await expect(catalog.dropDatabase("full")).rejects.toThrow();
await catalog.dropDatabase("missing", { ignoreMissing: true });
expect(requests[2].body).toEqual({
mode: "Fail",
behavior: "Restrict",
});
},
);
});
it("lazily traverses pages, including empty pages, and exposes cached state", async () => {
await withCatalog(
[
// biome-ignore lint/style/useNamingConvention: server wire format
[200, { namespaces: ["a", "b"], page_token: "empty" }],
// biome-ignore lint/style/useNamingConvention: server wire format
[200, { namespaces: [], page_token: "last" }],
// biome-ignore lint/style/useNamingConvention: server wire format
[200, { namespaces: ["c", "d"], page_token: "" }],
],
async (catalog, requests) => {
const names = catalog.listDatabases({ pageLimit: 2 });
expect(names[Symbol.asyncIterator]()).toBe(names);
expect(requests).toHaveLength(0);
expect(names.numPageResults()).toBe(0);
expect(names.pageToken()).toBeUndefined();
expect(await names.next()).toEqual({ done: false, value: "a" });
expect(names.numPageResults()).toBe(1);
expect(names.pageToken()).toBe("empty");
expect(await names.next()).toEqual({ done: false, value: "b" });
expect(requests).toHaveLength(1);
expect(await names.next()).toEqual({ done: false, value: "c" });
expect(requests).toHaveLength(3);
expect(names.pageToken()).toBeUndefined();
expect(names.numPageResults()).toBe(1);
const remaining = [];
for await (const name of names) remaining.push(name);
expect(remaining).toEqual(["d"]);
expect(names.numPageResults()).toBe(0);
expect(await names.next()).toEqual({ done: true, value: undefined });
expect(requests.map((request) => request.url)).toEqual([
"/v1/namespace/$/list?limit=2",
"/v1/namespace/$/list?limit=2&page_token=empty",
"/v1/namespace/$/list?limit=2&page_token=last",
]);
},
);
});
it("resumes from a saved token after draining the cache", async () => {
await withCatalog(
[
// biome-ignore lint/style/useNamingConvention: server wire format
[200, { namespaces: ["a", "b"], page_token: "next/token" }],
[200, { namespaces: ["c"] }],
],
async (catalog, requests) => {
const names = catalog.listDatabases({
pageToken: "start/token",
pageLimit: 2,
});
expect(names.pageToken()).toBe("start/token");
expect(await names.next()).toEqual({ done: false, value: "a" });
while (names.numPageResults() > 0) await names.next();
const resumed = catalog.listDatabases({
pageToken: names.pageToken(),
pageLimit: 2,
});
const results = [];
for await (const name of resumed) results.push(name);
expect(results).toEqual(["c"]);
expect(requests.map((request) => request.url)).toEqual([
"/v1/namespace/$/list?limit=2&page_token=start%2Ftoken",
"/v1/namespace/$/list?limit=2&page_token=next%2Ftoken",
]);
},
);
});
it("terminates on errors and retains the failed request token", async () => {
await withCatalog(
[
// biome-ignore lint/style/useNamingConvention: server wire format
[200, { namespaces: ["a"], page_token: "retry" }],
[400, {}],
[200, { namespaces: ["b"] }],
],
async (catalog, requests) => {
const names = catalog.listDatabases();
await names.next();
await expect(names.next()).rejects.toThrow();
expect(names.pageToken()).toBe("retry");
expect(names.numPageResults()).toBe(0);
expect(await names.next()).toEqual({ done: true, value: undefined });
expect(requests).toHaveLength(2);
const resumed = catalog.listDatabases({ pageToken: names.pageToken() });
expect(await resumed.next()).toEqual({ done: false, value: "b" });
expect(await resumed.next()).toEqual({ done: true, value: undefined });
},
);
});
it("handles an empty listing and concurrent advances", async () => {
await withCatalog(
[
[200, { namespaces: [] }],
[200, { namespaces: ["a", "b"] }],
],
async (catalog, requests) => {
const empty = catalog.listDatabases();
expect(await empty.next()).toEqual({ done: true, value: undefined });
expect(await empty.next()).toEqual({ done: true, value: undefined });
const names = catalog.listDatabases();
const pending = [names.next(), names.next(), names.next()];
expect(names.numPageResults()).toBe(0);
expect(names.pageToken()).toBeUndefined();
const results = await Promise.all(pending);
expect(
results
.filter((result) => !result.done)
.map((result) => result.value)
.sort(),
).toEqual(["a", "b"]);
expect(results.filter((result) => result.done)).toHaveLength(1);
expect(requests).toHaveLength(2);
},
);
});
it("validates endpoints and pagination", async () => {
await expect(connectCatalog("/tmp/catalog")).rejects.toThrow();
const catalog = await connectCatalog("http://127.0.0.1:1");
for (const limit of [
0,
-1,
1.5,
2147483648,
Number.NaN,
Number.POSITIVE_INFINITY,
]) {
expect(() => catalog.listDatabases({ pageLimit: limit })).toThrow(
"limit",
);
}
});
it.each([
"",
".",
"..",
"/db",
"db/",
"a//b",
"a/./b",
"a/../b",
"a b",
"a:b",
"a$b",
"a%b",
"a?b",
"a#b",
"a\\b",
"café",
"a\nb",
])(
"rejects invalid database name %j before sending requests",
async (name) => {
await withCatalog([], async (catalog, requests) => {
for (const method of [
"createDatabase",
"connectDatabase",
"dropDatabase",
] as const) {
await expect(catalog[method](name)).rejects.toThrow(
"Invalid database name",
);
}
expect(requests).toEqual([]);
});
},
);
});
nodejs/__test__/connection.test.ts
+419 -13
View File
@@ -1,10 +1,17 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { readdirSync } from "fs";
import { mkdirSync, readdirSync } from "fs";
import { join } from "path";
import { Field, Float64, Schema } from "apache-arrow";
import * as tmp from "tmp";
import { Connection, Table, connect } from "../lancedb";
import {
Connection,
ListTablesResponse,
Table,
connect,
connectNamespace,
} from "../lancedb";
import { LocalTable } from "../lancedb/table";
describe("when connecting", () => {
@@ -47,6 +54,15 @@ describe("given a connection", () => {
await db.close();
expect(db.isOpen()).toBe(false);
await expect(db.tableNames()).rejects.toThrow("Connection is closed");
await expect(db.listTables()).rejects.toThrow("Connection is closed");
await expect(db.renameTable("a", "b")).rejects.toThrow(
"Connection is closed",
);
});
it("should report renameTable as unsupported on an OSS connection", async () => {
await db.createTable("a", [{ id: 1 }]);
await expect(db.renameTable("a", "b")).rejects.toThrow(/not supported/);
});
it("should be able to create a table from an object arg `createTable(options)`, or args `createTable(name, data, options)`", async () => {
let tbl = await db.createTable("test", [{ id: 1 }, { id: 2 }]);
@@ -81,6 +97,16 @@ describe("given a connection", () => {
await db.createTable("test4", [{ id: 1 }, { id: 2 }]);
});
it("should return a completed job when dropping a local table", async () => {
await db.createTable("async-drop", [{ id: 1 }]);
const job = await db.dropTableAsync("async-drop");
expect(job.id).toBeNull();
await expect(job.status()).resolves.toBe("finished");
await job.wait();
await expect(db.tableNames()).resolves.toEqual([]);
});
it("should fail if creating table twice, unless overwrite is true", async () => {
let tbl = await db.createTable("test", [{ id: 1 }, { id: 2 }]);
await expect(tbl.countRows()).resolves.toBe(2);
@@ -111,6 +137,92 @@ describe("given a connection", () => {
expect(tables).toEqual(["b", "c"]);
});
it("should respect limit and page token when listing tables", async () => {
const db = await connect(tmpDir.name);
await db.createTable("b", [{ id: 1 }]);
await db.createTable("a", [{ id: 1 }]);
await db.createTable("c", [{ id: 1 }]);
const all = await db.listTables();
expect(all.tables).toEqual(["a", "b", "c"]);
expect(all.pageToken).toBeUndefined();
const first = await db.listTables({ limit: 1 });
expect(first.tables).toEqual(["a"]);
expect(first.pageToken).toBeDefined();
const second = await db.listTables({
limit: 1,
pageToken: first.pageToken,
});
expect(second.tables).toEqual(["b"]);
});
it("should visit every table exactly once when walking pages", async () => {
const db = await connect(tmpDir.name);
const created = ["a", "b", "c", "d", "e"];
for (const name of created) {
await db.createTable(name, [{ id: 1 }]);
}
const seen: string[] = [];
let pageToken: string | undefined = undefined;
do {
const page: ListTablesResponse = await db.listTables({
limit: 2,
pageToken,
});
seen.push(...page.tables);
pageToken = page.pageToken;
} while (pageToken);
expect(seen).toEqual(created);
});
it("should list all names and use a default page size of 100", async () => {
const names = Array.from(
{ length: 130 },
(_, i) => `t${i.toString().padStart(3, "0")}`,
);
for (const name of names) {
mkdirSync(join(tmpDir.name, `${name}.lance`));
}
await expect(db.tableNames()).resolves.toEqual(names);
await expect(db.tableNames({ limit: 0 })).resolves.toEqual([]);
const first = await db.listTables();
expect(first.tables).toEqual(names.slice(0, 100));
expect(first.pageToken).toBeDefined();
const second = await db.listTables({ pageToken: first.pageToken });
expect(second.tables).toEqual(names.slice(100));
expect(second.pageToken).toBeUndefined();
await expect(db.listTables({ limit: 200 })).resolves.toEqual({
tables: names,
pageToken: undefined,
});
await expect(db.listTables({ limit: 0 })).resolves.toEqual({
tables: [],
pageToken: undefined,
});
});
it("should list tables in a namespace", async () => {
const db = await connect(tmpDir.name, {
// biome-ignore lint/style/useNamingConvention: opaque backend property key, must match Rust
namespaceClientProperties: { manifest_enabled: "true" },
});
await db.createNamespace(["child"]);
await db.createTable("nested", [{ id: 1 }], ["child"]);
await expect(db.listTables(["child"])).resolves.toEqual(
expect.objectContaining({ tables: ["nested"] }),
);
await expect(db.listTables()).resolves.toEqual(
expect.objectContaining({ tables: [] }),
);
});
it("should create tables in v2 mode", async () => {
const db = await connect(tmpDir.name);
const data = [...Array(10000).keys()].map((i) => ({ id: i }));
@@ -163,18 +275,22 @@ describe("given a connection", () => {
let manifestDir =
tmpDir.name + "/test_manifest_paths_v2_empty.lance/_versions";
readdirSync(manifestDir).forEach((file) => {
expect(file).toMatch(/^\d{20}\.manifest$/);
});
readdirSync(manifestDir)
.filter((f) => f.endsWith(".manifest"))
.forEach((file) => {
expect(file).toMatch(/^\d{20}\.manifest$/);
});
table = (await db.createTable("test_manifest_paths_v2", [{ id: 1 }], {
enableV2ManifestPaths: true,
})) as LocalTable;
expect(await table.usesV2ManifestPaths()).toBe(true);
manifestDir = tmpDir.name + "/test_manifest_paths_v2.lance/_versions";
readdirSync(manifestDir).forEach((file) => {
expect(file).toMatch(/^\d{20}\.manifest$/);
});
readdirSync(manifestDir)
.filter((f) => f.endsWith(".manifest"))
.forEach((file) => {
expect(file).toMatch(/^\d{20}\.manifest$/);
});
});
it("should be able to migrate tables to the V2 manifest paths", async () => {
@@ -191,15 +307,305 @@ describe("given a connection", () => {
const manifestDir =
tmpDir.name + "/test_manifest_path_migration.lance/_versions";
readdirSync(manifestDir).forEach((file) => {
expect(file).toMatch(/^\d\.manifest$/);
});
readdirSync(manifestDir)
.filter((f) => f.endsWith(".manifest"))
.forEach((file) => {
expect(file).toMatch(/^\d\.manifest$/);
});
await table.migrateManifestPathsV2();
expect(await table.usesV2ManifestPaths()).toBe(true);
readdirSync(manifestDir).forEach((file) => {
expect(file).toMatch(/^\d{20}\.manifest$/);
readdirSync(manifestDir)
.filter((f) => f.endsWith(".manifest"))
.forEach((file) => {
expect(file).toMatch(/^\d{20}\.manifest$/);
});
});
});
describe("clone table functionality", () => {
let tmpDir: tmp.DirResult;
let db: Connection;
beforeEach(async () => {
tmpDir = tmp.dirSync({ unsafeCleanup: true });
db = await connect(tmpDir.name);
});
afterEach(() => tmpDir.removeCallback());
it("should clone a table with latest version (default behavior)", async () => {
// Create source table with some data
const data = [
{ id: 1, text: "hello", vector: [1.0, 2.0] },
{ id: 2, text: "world", vector: [3.0, 4.0] },
];
const sourceTable = await db.createTable("source", data);
// Add more data to create a new version
const moreData = [{ id: 3, text: "test", vector: [5.0, 6.0] }];
await sourceTable.add(moreData);
// Clone the table (should get latest version with 3 rows)
const sourceUri = `${tmpDir.name}/source.lance`;
const clonedTable = await db.cloneTable("cloned", sourceUri);
// Verify cloned table has all 3 rows
expect(await clonedTable.countRows()).toBe(3);
expect((await db.tableNames()).includes("cloned")).toBe(true);
});
it("should clone a table from a specific version", async () => {
// Create source table with initial data
const data = [
{ id: 1, text: "hello", vector: [1.0, 2.0] },
{ id: 2, text: "world", vector: [3.0, 4.0] },
];
const sourceTable = await db.createTable("source", data);
// Get the initial version
const initialVersion = await sourceTable.version();
// Add more data to create a new version
const moreData = [{ id: 3, text: "test", vector: [5.0, 6.0] }];
await sourceTable.add(moreData);
// Verify source now has 3 rows
expect(await sourceTable.countRows()).toBe(3);
// Clone from the initial version (should have only 2 rows)
const sourceUri = `${tmpDir.name}/source.lance`;
const clonedTable = await db.cloneTable("cloned", sourceUri, {
sourceVersion: initialVersion,
});
// Verify cloned table has only the initial 2 rows
expect(await clonedTable.countRows()).toBe(2);
});
it("should clone a table from a tagged version", async () => {
// Create source table with initial data
const data = [
{ id: 1, text: "hello", vector: [1.0, 2.0] },
{ id: 2, text: "world", vector: [3.0, 4.0] },
];
const sourceTable = await db.createTable("source", data);
// Create a tag for the current version
const tags = await sourceTable.tags();
await tags.create("v1.0", await sourceTable.version());
// Add more data after the tag
const moreData = [{ id: 3, text: "test", vector: [5.0, 6.0] }];
await sourceTable.add(moreData);
// Verify source now has 3 rows
expect(await sourceTable.countRows()).toBe(3);
// Clone from the tagged version (should have only 2 rows)
const sourceUri = `${tmpDir.name}/source.lance`;
const clonedTable = await db.cloneTable("cloned", sourceUri, {
sourceTag: "v1.0",
});
// Verify cloned table has only the tagged version's 2 rows
expect(await clonedTable.countRows()).toBe(2);
});
it("should fail when attempting deep clone", async () => {
// Create source table with some data
const data = [
{ id: 1, text: "hello", vector: [1.0, 2.0] },
{ id: 2, text: "world", vector: [3.0, 4.0] },
];
await db.createTable("source", data);
// Try to create a deep clone (should fail)
const sourceUri = `${tmpDir.name}/source.lance`;
await expect(
db.cloneTable("cloned", sourceUri, { isShallow: false }),
).rejects.toThrow("Deep clone is not yet implemented");
});
});
describe("namespaces", () => {
let tmpDir: tmp.DirResult;
let db: Connection;
beforeEach(async () => {
tmpDir = tmp.dirSync({ unsafeCleanup: true });
// The local DirectoryNamespace backend only supports child namespaces
// when manifest mode is enabled (see lance-namespace-impls/src/dir.rs).
db = await connect(tmpDir.name, {
// biome-ignore lint/style/useNamingConvention: opaque backend property key, must match Rust
namespaceClientProperties: { manifest_enabled: "true" },
});
});
afterEach(() => tmpDir.removeCallback());
it("should create and describe a namespace", async () => {
await db.createNamespace(["myns"]);
const desc = await db.describeNamespace(["myns"]);
expect(desc).toBeDefined();
});
it("should list namespaces created at the root", async () => {
await db.createNamespace(["alpha"]);
await db.createNamespace(["beta"]);
const list = await db.listNamespaces();
expect(list.namespaces).toEqual(expect.arrayContaining(["alpha", "beta"]));
});
it("should list child namespaces under a parent", async () => {
await db.createNamespace(["parent"]);
await db.createNamespace(["parent", "child"]);
const list = await db.listNamespaces(["parent"]);
expect(list.namespaces).toContain("child");
});
it("should drop a namespace", async () => {
await db.createNamespace(["ephemeral"]);
await db.dropNamespace(["ephemeral"]);
const list = await db.listNamespaces();
expect(list.namespaces).not.toContain("ephemeral");
});
it("should raise an error on any namespace op after close", async () => {
await db.close();
await expect(db.describeNamespace(["foo"])).rejects.toThrow(
"Connection is closed",
);
await expect(db.listNamespaces()).rejects.toThrow("Connection is closed");
await expect(db.createNamespace(["foo"])).rejects.toThrow(
"Connection is closed",
);
await expect(db.dropNamespace(["foo"])).rejects.toThrow(
"Connection is closed",
);
});
it("should raise an understandable error when describing a non-existent namespace", async () => {
await expect(db.describeNamespace(["does-not-exist"])).rejects.toThrow(
/not found/i,
);
});
it("should raise an error when creating a namespace that already exists", async () => {
await db.createNamespace(["dup"]);
await expect(db.createNamespace(["dup"])).rejects.toThrow();
});
it("should reject an unrecognized createNamespace mode with a clear error", async () => {
await expect(
// biome-ignore lint/suspicious/noExplicitAny: deliberately bypass TS to test runtime validation
db.createNamespace(["x"], { mode: "frobnicate" as any }),
).rejects.toThrow(/Invalid mode 'frobnicate'/);
});
it("should reject an unrecognized dropNamespace mode with a clear error", async () => {
await db.createNamespace(["x"]);
await expect(
// biome-ignore lint/suspicious/noExplicitAny: deliberately bypass TS to test runtime validation
db.dropNamespace(["x"], { mode: "frobnicate" as any }),
).rejects.toThrow(/Invalid mode 'frobnicate'/);
});
it("should reject an unrecognized dropNamespace behavior with a clear error", async () => {
await db.createNamespace(["x"]);
await expect(
// biome-ignore lint/suspicious/noExplicitAny: deliberately bypass TS to test runtime validation
db.dropNamespace(["x"], { behavior: "frobnicate" as any }),
).rejects.toThrow(/Invalid behavior 'frobnicate'/);
});
});
describe("connectNamespace", () => {
let tmpDir: tmp.DirResult;
beforeEach(() => {
tmpDir = tmp.dirSync({ unsafeCleanup: true });
});
afterEach(() => tmpDir.removeCallback());
it("connects via the dir implementation and supports table ops", async () => {
const db = await connectNamespace("dir", { root: tmpDir.name });
await db.createTable("users", [{ id: 1 }, { id: 2 }]);
await expect(db.tableNames()).resolves.toContain("users");
});
it("throws a clear error when implName is empty", async () => {
await expect(connectNamespace("", {})).rejects.toThrow(
"implName must be a non-empty string",
);
});
it("throws when the namespace implementation is unknown", async () => {
await expect(connectNamespace("not-a-real-impl", {})).rejects.toThrow();
});
it("passes storage options through to the namespace", async () => {
const db = await connectNamespace(
"dir",
{ root: tmpDir.name },
{ storageOptions: { newTableDataStorageVersion: "stable" } },
);
await db.createTable("plumbing", [{ id: 1 }]);
await expect(db.tableNames()).resolves.toContain("plumbing");
});
it("supports child namespaces when manifestEnabled is true on the dir config", async () => {
const writer = await connectNamespace("dir", {
root: tmpDir.name,
manifestEnabled: true,
});
await writer.createNamespace(["analytics"]);
await writer.createTable("orders", [{ id: 1 }, { id: 2 }], ["analytics"]);
await writer.close();
const reader = await connectNamespace("dir", {
root: tmpDir.name,
manifestEnabled: true,
});
await expect(reader.tableNames(["analytics"])).resolves.toContain("orders");
const orders = await reader.openTable("orders", ["analytics"]);
await expect(orders.countRows()).resolves.toBe(2);
});
it("merges extraProperties into the dir config and is overridden by typed fields", async () => {
// Two observable assertions:
// - Typed `root` overrides extraProperties.root: createTable would fail
// under the bogus path if the override didn't happen.
// - extraProperties.manifest_enabled="false" is honored end-to-end. Child
// namespaces require manifest mode (default true), so explicitly
// disabling it via extraProperties must make createNamespace reject. If
// extraProperties pass-through were silently broken, the default would
// let createNamespace succeed.
const db = await connectNamespace("dir", {
root: tmpDir.name,
extraProperties: {
root: "/should/be/overridden",
// biome-ignore lint/style/useNamingConvention: backend property key
manifest_enabled: "false",
},
});
await db.createTable("base", [{ id: 1 }]);
await expect(db.tableNames()).resolves.toContain("base");
await expect(db.createNamespace(["analytics"])).rejects.toThrow();
});
it("flows unknown top-level keys through when implName is dynamic (no silent drop)", async () => {
// Routes via the third overload because `impl` is `string`, not the
// literal `"dir"`. The dispatcher still notices the runtime value is
// "dir", but unknown keys like `manifest_enabled` must not be silently
// dropped during the conversion.
//
// Asserting a *negative* outcome (manifest disabled -> createNamespace
// rejects) is required for observability, since the backend default for
// `manifest_enabled` is true.
const impl: string = "dir";
const db = await connectNamespace(impl, {
root: tmpDir.name,
// biome-ignore lint/style/useNamingConvention: backend property key
manifest_enabled: "false",
});
await expect(db.createNamespace(["mixed"])).rejects.toThrow();
});
});
nodejs/__test__/embedding.test.ts
+214
View File
@@ -11,8 +11,11 @@ import {
Float16,
Float32,
Float64,
Int32,
Schema,
Utf8,
fromDataToBuffer,
tableFromIPC,
} from "../lancedb/arrow";
import { EmbeddingFunction, LanceSchema } from "../lancedb/embedding";
import { getRegistry, register } from "../lancedb/embedding/registry";
@@ -184,6 +187,115 @@ describe("embedding functions", () => {
const vector0 = JSON.parse(JSON.stringify(arr[0].vector));
expect(vector0).toEqual([1, 2, 3]);
});
it("should append multiple Python embeddings with the same alias", async () => {
@register("python-mock")
// biome-ignore lint/correctness/noUnusedVariables: the decorator registers this class
class MockEmbeddingFunction extends EmbeddingFunction<string> {
ndims() {
return 3;
}
embeddingDataType(): Float {
return new Float32();
}
async computeQueryEmbeddings(_data: string) {
return [1, 2, 3];
}
async computeSourceEmbeddings(data: string[]) {
return data.map((value) =>
value === "hello world" ? [1, 2, 3] : [4, 5, 6],
);
}
}
const metadata = new Map([
[
"embedding_functions",
'[{"source_column":"text1","vector_column":"vector1","name":"python-mock","model":{}},{"source_column":"text2","vector_column":"vector2","name":"python-mock","model":{}}]',
],
]);
const schema = new Schema(
[
new Field("text1", new Utf8(), true),
new Field("text2", new Utf8(), true),
new Field(
"vector1",
new FixedSizeList(3, new Field("item", new Float32(), true)),
true,
),
new Field(
"vector2",
new FixedSizeList(3, new Field("item", new Float32(), true)),
true,
),
],
metadata,
);
const db = await connect(tmpDir.name);
const table = await db.createEmptyTable("test", schema);
await table.add([{ text1: "hello world", text2: "goodbye world" }]);
const rows = await table.query().toArray();
expect(JSON.parse(JSON.stringify(rows[0].vector1))).toEqual([1, 2, 3]);
expect(JSON.parse(JSON.stringify(rows[0].vector2))).toEqual([4, 5, 6]);
});
it("should append generated vectors to a non-nullable schema", async () => {
@register("non_nullable_schema_test")
class MockEmbeddingFunction extends EmbeddingFunction<string> {
ndims() {
return 3;
}
embeddingDataType(): Float {
return new Float64();
}
async computeSourceEmbeddings(data: string[]) {
return data.map(() => [1, 2, 3]);
}
}
const schema = new Schema([
new Field("id", new Int32()),
new Field("text", new Utf8()),
new Field("type", new Utf8()),
new Field(
"vector",
new FixedSizeList(3, new Field("item", new Float64())),
),
]);
const func = new MockEmbeddingFunction();
const db = await connect(tmpDir.name);
const table = await db.createEmptyTable("test_non_nullable", schema, {
embeddingFunction: {
function: func,
sourceColumn: "text",
},
});
const data = [
{ id: 1, text: "Carrot", type: "vegetable" },
{ id: 2, text: "Apple", type: "fruit" },
];
const buffer = await fromDataToBuffer(
data,
undefined,
await table.schema(),
);
const generatedTable = tableFromIPC(buffer);
const vectorField = generatedTable.schema.fields.find(
(field) => field.name === "vector",
);
expect(vectorField?.nullable).toBe(false);
await table.add(data);
const rows = await table.query().toArray();
expect(rows).toHaveLength(2);
for (const row of rows) {
expect([...row.vector]).toEqual([1, 2, 3]);
}
});
it("should error when appending to a table with an unregistered embedding function", async () => {
@register("mock")
class MockEmbeddingFunction extends EmbeddingFunction<string> {
@@ -256,6 +368,60 @@ describe("embedding functions", () => {
expect(actual).toHaveProperty("text");
});
it("should handle undefined vector field with embedding function correctly", async () => {
@register("undefined_test")
class MockEmbeddingFunction extends EmbeddingFunction<string> {
ndims() {
return 3;
}
embeddingDataType(): Float {
return new Float32();
}
async computeQueryEmbeddings(_data: string) {
return [1, 2, 3];
}
async computeSourceEmbeddings(data: string[]) {
return Array.from({ length: data.length }).fill([
1, 2, 3,
]) as number[][];
}
}
const func = getRegistry()
.get<MockEmbeddingFunction>("undefined_test")!
.create();
const schema = new Schema([
new Field("text", new Utf8(), true),
new Field(
"vector",
new FixedSizeList(3, new Field("item", new Float32(), true)),
true,
),
]);
const db = await connect(tmpDir.name);
const table = await db.createEmptyTable("test_undefined", schema, {
embeddingFunction: {
function: func,
sourceColumn: "text",
vectorColumn: "vector",
},
});
// Test that undefined, null, and omitted vector fields all work
await table.add([{ text: "test1", vector: undefined }]);
await table.add([{ text: "test2", vector: null }]);
await table.add([{ text: "test3" }]);
const rows = await table.query().toArray();
expect(rows.length).toBe(3);
// All rows should have vectors computed by the embedding function
for (const row of rows) {
expect(row.vector).toBeDefined();
expect(JSON.parse(JSON.stringify(row.vector))).toEqual([1, 2, 3]);
}
});
test.each([new Float16(), new Float32(), new Float64()])(
"should be able to provide manual embeddings with multiple float datatype",
async (floatType) => {
@@ -373,4 +539,52 @@ describe("embedding functions", () => {
expect(stringSchema3).toEqual(stringExpectedSchema);
},
);
test("parses one function writing several vector columns", async () => {
class MockEmbeddingFunction extends EmbeddingFunction<string> {
ndims() {
return 3;
}
embeddingDataType(): Float {
return new Float32();
}
async computeQueryEmbeddings(_data: string) {
return [1, 2, 3];
}
async computeSourceEmbeddings(data: string[]) {
return Array.from({ length: data.length }).fill([
1, 2, 3,
]) as number[][];
}
}
const registry = getRegistry();
registry.register("multi_output_mock")(MockEmbeddingFunction);
// A materialized view can project one source vector column under two
// names, so a table's configuration names the same function twice.
const parsed = await registry.parseFunctions(
new Map([
[
"embedding_functions",
JSON.stringify([
{
name: "multi_output_mock",
sourceColumn: "text",
vectorColumn: "vector_a",
model: {},
},
{
name: "multi_output_mock",
sourceColumn: "text",
vectorColumn: "vector_b",
model: {},
},
]),
],
]),
);
expect(
[...parsed.values()].map(({ vectorColumn }) => vectorColumn).sort(),
).toEqual(["vector_a", "vector_b"]);
});
});
nodejs/__test__/embedding_registry.test.ts
+95
View File
@@ -0,0 +1,95 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { execFileSync } from "node:child_process";
import { resolve } from "node:path";
import type { OpenAIEmbeddingFunction } from "../lancedb/embedding/openai";
import type { EmbeddingFunctionRegistry } from "../lancedb/embedding/registry";
type EmbeddingModule = typeof import("../lancedb/embedding");
type OpenAIModule = typeof import("../lancedb/embedding/openai");
type RegistryModule = typeof import("../lancedb/embedding/registry");
describe("embedding function registry", () => {
const registries: EmbeddingFunctionRegistry[] = [];
afterEach(() => {
for (const registry of registries) {
registry.reset();
}
registries.length = 0;
});
it("defers built-in providers until the public registry API is used", () => {
jest.isolateModules(() => {
const embedding = require("../lancedb/embedding") as EmbeddingModule;
const { getRegistry: getInternalRegistry } =
require("../lancedb/embedding/registry") as RegistryModule;
const registry = getInternalRegistry();
registries.push(registry);
expect(registry.length()).toBe(0);
expect(embedding.getRegistry()).toBe(registry);
expect(registry.get("openai")).toBeDefined();
expect(registry.get("huggingface")).toBeDefined();
});
});
it("preserves automatic FTS search in a fresh process", () => {
execFileSync(
process.execPath,
[resolve(__dirname, "fixtures", "auto_fts_search.cjs")],
{ stdio: "pipe" },
);
});
it("shares registrations across duplicated provider module graphs", () => {
let registeringRegistry: EmbeddingFunctionRegistry | undefined;
let latestOpenAIConstructor: typeof OpenAIEmbeddingFunction | undefined;
jest.isolateModules(() => {
require("../lancedb/embedding/openai");
const { getRegistry } =
require("../lancedb/embedding/registry") as RegistryModule;
registeringRegistry = getRegistry();
registries.push(registeringRegistry);
expect(registeringRegistry.get("openai")).toBeDefined();
});
expect(() => {
jest.isolateModules(() => {
const { OpenAIEmbeddingFunction } =
require("../lancedb/embedding/openai") as OpenAIModule;
latestOpenAIConstructor = OpenAIEmbeddingFunction;
const { getRegistry } =
require("../lancedb/embedding/registry") as RegistryModule;
registries.push(getRegistry());
});
}).not.toThrow();
const previousApiKey = process.env.OPENAI_API_KEY;
process.env.OPENAI_API_KEY = "test";
try {
const latestOpenAI = registeringRegistry!
.get<OpenAIEmbeddingFunction>("openai")!
.create();
expect(latestOpenAI).toBeInstanceOf(latestOpenAIConstructor!);
} finally {
if (previousApiKey === undefined) {
delete process.env.OPENAI_API_KEY;
} else {
process.env.OPENAI_API_KEY = previousApiKey;
}
}
jest.isolateModules(() => {
const { getRegistry } =
require("../lancedb/embedding") as EmbeddingModule;
const publicRegistry = getRegistry();
registries.push(publicRegistry);
expect(publicRegistry).toBe(registeringRegistry);
expect(publicRegistry.get("openai")).toBeDefined();
});
});
});
nodejs/__test__/fixtures/auto_fts_search.cjs
+33
View File
@@ -0,0 +1,33 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
const assert = require("node:assert/strict");
const tmp = require("tmp");
const { connect, embedding, Index } = require("../../dist");
const { getRegistry } = require("../../dist/embedding/registry");
async function main() {
assert.equal(typeof embedding.getRegistry, "function");
assert.equal(getRegistry().length(), 0);
assert.equal(embedding.getRegistry(), getRegistry());
assert.equal(getRegistry().length(), 2);
const dir = tmp.dirSync({ unsafeCleanup: true });
let db;
try {
db = await connect(dir.name);
const table = await db.createTable("docs", [{ text: "hello world" }]);
await table.createIndex("text", { config: Index.fts() });
const rows = await table.search("hello").toArray();
assert.equal(rows[0].text, "hello world");
} finally {
db?.close();
dir.removeCallback();
}
}
main().catch((error) => {
console.error(error);
process.exitCode = 1;
});
nodejs/__test__/fixtures/oauth_browser.cmd
+3
View File
@@ -0,0 +1,3 @@
@rem SPDX-License-Identifier: Apache-2.0
@rem SPDX-FileCopyrightText: Copyright The LanceDB Authors
@exit /b 0
nodejs/__test__/materialized_view.test.ts
+181
View File
@@ -0,0 +1,181 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import * as tmp from "tmp";
import { Connection, connect } from "../lancedb";
import {
DEFINITION_META_KEY,
definitionFromMetadata,
} from "../lancedb/materialized_view";
describe("materialized views", () => {
let tmpDir: tmp.DirResult;
let db: Connection;
beforeEach(async () => {
tmpDir = tmp.dirSync({ unsafeCleanup: true });
db = await connect(tmpDir.name);
await db.createTable("people", [
{ name: "ada", age: 36 },
{ name: "kid", age: 7 },
{ name: "grace", age: 85 },
]);
});
afterEach(() => tmpDir.removeCallback());
it("reads stored queries and legacy layouts", () => {
const read = (stored: string) =>
definitionFromMetadata(new Map([[DEFINITION_META_KEY, stored]]), "v");
const query =
"SELECT id, c.chunk FROM ns.docs, UNNEST(chunks) AS c WHERE id > 1";
expect(read(`{"format":1,"query":${JSON.stringify(query)}}`).query).toBe(
query,
);
// The structured layout written before the format number reads as the
// query it described, under either of its kind tags.
expect(
read(
'{"kind":"namespaced_select","source_table":"people","source_namespace":["ns"],' +
'"projections":[{"output":"name","expression":"`name`"},' +
'{"output":"Shout","expression":"upper(name)"}],"filter":"age >= 18","limit":42}',
).query,
).toBe(
"SELECT `name`, upper(name) AS `Shout` FROM ns.people WHERE age >= 18 LIMIT 42",
);
expect(read('{"kind":"select","source_table":"people"}').query).toBe(
"SELECT * FROM people",
);
// A newer writer's layout is reported, never guessed at.
for (const newer of [
`{"format":3,"query":${JSON.stringify(query)}}`,
'{"kind":"select_v3","source_table":"people"}',
]) {
expect(() => read(newer)).toThrow(/cannot refresh/);
}
});
it("rejects a stored limit a number cannot carry", () => {
const big = new Map([
[
DEFINITION_META_KEY,
'{"kind":"select","source_table":"people","limit":9007199254740993}',
],
]);
expect(() => definitionFromMetadata(big, "v")).toThrow(
/too large to represent exactly/,
);
});
it("creates, refreshes and queries a view", async () => {
const view = await db.createMaterializedView("adults", "people", {
select: ["name", ["shout", "upper(name)"]],
where: "age >= 18",
});
expect(view.name).toBe("adults");
expect(await view.table().countRows()).toBe(2);
const rows = await view.table().query().toArray();
expect(rows.map((r) => r.shout).sort()).toEqual(["ADA", "GRACE"]);
});
it("round-trips the definition", async () => {
await db.createMaterializedView("adults", "people", {
where: "age >= 18",
});
const view = await db.openMaterializedView("adults");
const definition = await view.definition();
expect(definition.query).toBe(
"SELECT name, age FROM people WHERE age >= 18",
);
});
it("fully refreshes after an append", async () => {
const view = await db.createMaterializedView("copy", "people", {
withNoData: true,
});
await view.refresh();
const people = await db.openTable("people");
await people.add([{ name: "alan", age: 41 }]);
const result = await view.refresh();
expect(result.mode).toBe("rebuild");
expect(Number(result.rowsWritten)).toBe(4);
expect(await view.table().countRows()).toBe(4);
expect((await view.refresh()).mode).toBe("rebuild");
});
it("lists views and rejects non-views", async () => {
await db.createMaterializedView("adults", "people", {
where: "age >= 18",
});
expect(await collect(db.listMaterializedViews())).toEqual(["adults"]);
await expect(db.openMaterializedView("people")).rejects.toThrow(
"not a materialized view",
);
await expect(db.dropMaterializedView("people")).rejects.toThrow(
"not a materialized view",
);
await db.dropMaterializedView("adults");
expect(await collect(db.listMaterializedViews())).toEqual([]);
});
it("returns a job when dropping a view asynchronously", async () => {
await db.createMaterializedView("adults", "people");
const job = await db.dropMaterializedViewAsync("adults");
expect(job.id).toBeNull();
await job.wait();
expect(await collect(db.listMaterializedViews())).toEqual([]);
});
it("rejects an invalid expression at create time", async () => {
await expect(
db.createMaterializedView("bad", "people", {
select: [["x", "missing + 1"]],
}),
).rejects.toThrow("missing");
});
it("rejects invalid numeric options before creating anything", async () => {
for (const limit of [-5, 1.5, Infinity, NaN]) {
await expect(
db.createMaterializedView("bad", "people", { limit }),
).rejects.toThrow("non-negative integer");
}
expect(await collect(db.listMaterializedViews())).toEqual([]);
const view = await db.createMaterializedView("copy", "people");
for (const sourceVersion of [-1, 1.5, Infinity, NaN]) {
await expect(view.refresh({ sourceVersion })).rejects.toThrow(
"non-negative integer",
);
}
});
it("quotes bare select names", async () => {
await db.createTable("odd_names", [{ "order item": "widget" }]);
const view = await db.createMaterializedView("quoted", "odd_names", {
select: ["order item"],
withNoData: true,
});
const result = await view.refresh();
expect(Number(result.rowsWritten)).toBe(1);
});
it("does not require stable row ids on the source", async () => {
await db.createTable("plain", [{ x: 1 }]);
const view = await db.createMaterializedView("v", "plain");
expect(await view.table().countRows()).toBe(1);
});
});
async function collect<T>(items: AsyncIterable<T>): Promise<T[]> {
const result: T[] = [];
for await (const item of items) result.push(item);
return result;
}
nodejs/__test__/oauth.test.ts
+245
View File
@@ -0,0 +1,245 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import * as fs from "fs";
import * as http from "http";
import { execFileSync } from "node:child_process";
import * as os from "os";
import * as path from "path";
import { OAuthConfig, OAuthFlowType, OAuthSession } from "../lancedb/oauth";
function tempCacheDir(): string {
return fs.mkdtempSync(path.join(os.tmpdir(), "lancedb-oauth-cache-"));
}
function deviceConfig(issuerUrl: string, cacheDir: string): OAuthConfig {
return {
issuerUrl,
clientId: "client-id",
scopes: ["openid"],
flow: OAuthFlowType.DeviceCode,
tokenCache: { cacheDir },
};
}
describe("OAuthSession", () => {
beforeAll(() => {
// Child processes inherit the real environment, just as Rust reads it.
// Fail before login if the browser override only exists in Jest's sandbox.
const browser = execFileSync(
process.execPath,
["-p", "process.env.LANCEDB_OAUTH_BROWSER ?? ''"],
{ encoding: "utf8" },
).trim();
expect(browser).not.toBe("");
expect(browser).toBe(process.env.LANCEDB_OAUTH_BROWSER);
});
it("reports an absent session and logout is idempotent", async () => {
const cacheDir = tempCacheDir();
const session = new OAuthSession(
deviceConfig("https://issuer.example.com", cacheDir),
);
const status = await session.status();
expect(status.refreshable).toBe(false);
expect(status.issuerUrl).toBe("https://issuer.example.com");
expect(status.clientId).toBe("client-id");
expect(status.scopes).toEqual(["openid"]);
expect(status.flow).toBe("device_code");
expect(status.obtainedAt).toBeUndefined();
const logout = await session.logout();
expect(logout.removed).toBe(false);
});
it("requires token cache options", () => {
const config: OAuthConfig = {
issuerUrl: "https://issuer.example.com",
clientId: "client-id",
scopes: ["openid"],
flow: OAuthFlowType.DeviceCode,
};
expect(() => new OAuthSession(config)).toThrow(/token/);
});
it("rejects azure managed identity persistence", () => {
const config: OAuthConfig = {
issuerUrl: "https://login.microsoftonline.com/tenant/v2.0",
clientId: "app-id",
scopes: ["api://app/.default"],
flow: OAuthFlowType.AzureManagedIdentity,
tokenCache: { cacheDir: tempCacheDir() },
};
expect(() => new OAuthSession(config)).toThrow(/AzureManagedIdentity/);
});
it.each([
{},
{
resource: "https://api.example.com/a?x=1&y=two",
audience: "audience + & / ü",
},
])(
"logs in via device flow with target %j, caches, and logs out",
async (target) => {
const server = new MockIdp();
await server.start();
try {
const cacheDir = tempCacheDir();
const issuerUrl = server.issuerUrl();
const config = { ...deviceConfig(issuerUrl, cacheDir), ...target };
const session = new OAuthSession(config);
const status = await session.login();
expect(status.refreshable).toBe(true);
expect(status.resource).toBe(config.resource);
expect(status.audience).toBe(config.audience);
expect(status.obtainedAt).toBeGreaterThan(0);
expect(server.state.deviceAuthorizations).toBe(1);
// An independent session (a fresh "process") sees the cached login.
const other = new OAuthSession(config);
const cached = await other.status();
expect(cached.refreshable).toBe(true);
const logout = await other.logout();
expect(logout.removed).toBe(true);
const again = await session.logout();
expect(again.removed).toBe(false);
expect((await session.status()).refreshable).toBe(false);
// Only the initial login used the interactive device flow.
expect(server.state.deviceAuthorizations).toBe(1);
expect(server.state.refreshGrants).toBe(0);
expect(server.requests).toHaveLength(2);
for (const params of server.requests) {
expect(params.getAll("resource")).toEqual(
config.resource === undefined ? [] : [config.resource],
);
expect(params.getAll("audience")).toEqual(
config.audience === undefined ? [] : [config.audience],
);
}
} finally {
server.close();
}
},
15000,
);
});
/** Mock IdP with discovery, device authorization, and rotating refresh. */
class MockIdp {
readonly requests: URLSearchParams[] = [];
readonly state = {
deviceAuthorizations: 0,
refreshGrants: 0,
accessTokensIssued: 0,
currentRefresh: null as string | null,
};
private server?: http.Server;
private port = 0;
issuerUrl(): string {
return `http://127.0.0.1:${this.port}`;
}
async start(): Promise<void> {
const server = http.createServer((req, res) => {
const chunks: Buffer[] = [];
req.on("data", (chunk) => chunks.push(chunk));
req.on("end", () => {
const body = Buffer.concat(chunks).toString();
const params = new URLSearchParams(body);
this.handle(req.url ?? "", params, res);
});
});
await new Promise<void>((resolve) => {
server.listen(0, "127.0.0.1", () => resolve());
});
const address = server.address();
if (address && typeof address === "object") {
this.port = address.port;
}
this.server = server;
}
private handle(
url: string,
params: URLSearchParams,
res: http.ServerResponse,
): void {
const respond = (status: number, payload: unknown): void => {
const body = JSON.stringify(payload);
res.writeHead(status, {
"Content-Type": "application/json",
"Content-Length": Buffer.byteLength(body),
});
res.end(body);
};
if (url === "/.well-known/openid-configuration") {
respond(200, {
// biome-ignore lint/style/useNamingConvention: OAuth wire format
token_endpoint: `${this.issuerUrl()}/token`,
// biome-ignore lint/style/useNamingConvention: OAuth wire format
device_authorization_endpoint: `${this.issuerUrl()}/device`,
});
return;
}
if (url === "/device" || url === "/token") {
this.requests.push(params);
}
if (url === "/device") {
this.state.deviceAuthorizations += 1;
respond(200, {
// biome-ignore lint/style/useNamingConvention: OAuth wire format
device_code: "device-code",
// biome-ignore lint/style/useNamingConvention: OAuth wire format
user_code: "ABCD-EFGH",
// biome-ignore lint/style/useNamingConvention: OAuth wire format
verification_uri: `${this.issuerUrl()}/verify`,
// biome-ignore lint/style/useNamingConvention: OAuth wire format
expires_in: 60,
interval: 1,
});
return;
}
if (url === "/token") {
const grantType = params.get("grant_type") ?? "";
if (grantType === "refresh_token") {
this.state.refreshGrants += 1;
if (params.get("refresh_token") !== this.state.currentRefresh) {
respond(400, { error: "invalid_grant" });
return;
}
} else if (!grantType.includes("device_code")) {
respond(400, { error: "unsupported_grant_type" });
return;
}
this.state.accessTokensIssued += 1;
const number = this.state.accessTokensIssued;
const refresh = `refresh-${number}`;
this.state.currentRefresh = refresh;
respond(200, {
// biome-ignore lint/style/useNamingConvention: OAuth wire format
access_token: `access-${number}`,
// biome-ignore lint/style/useNamingConvention: OAuth wire format
refresh_token: refresh,
// biome-ignore lint/style/useNamingConvention: OAuth wire format
expires_in: 3600,
});
return;
}
respond(404, {});
}
close(): void {
this.server?.close();
}
}
nodejs/__test__/otel.test.ts
+114
View File
@@ -0,0 +1,114 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import {
MeterProvider,
type MetricData,
MetricReader,
} from "@opentelemetry/sdk-metrics";
import * as tmp from "tmp";
import { connect, instrumentLanceDbMetrics } from "../lancedb";
// snapshotLancedbMetrics is internal plumbing (not part of the public API), so
// it is imported from the native module rather than the package entry point.
import { snapshotLancedbMetrics } from "../lancedb/native";
// The metrics recorder is process-global and installed once, so the whole
// bridge is exercised in a single test to avoid cross-test global-state coupling.
// A minimal pull-based reader whose `collect()` we drive directly, invoking the
// observable-instrument callbacks. `@opentelemetry/sdk-metrics` ships no
// in-memory reader, so we subclass the abstract base.
class TestMetricReader extends MetricReader {
protected async onForceFlush(): Promise<void> {
// no-op: collection is driven directly via collect()
}
protected async onShutdown(): Promise<void> {
// no-op: nothing to release
}
}
async function metricsByName(
reader: TestMetricReader,
): Promise<Map<string, MetricData>> {
const collected = await reader.collect();
const result = new Map<string, MetricData>();
for (const scope of collected.resourceMetrics.scopeMetrics) {
for (const metric of scope.metrics) {
result.set(metric.descriptor.name, metric);
}
}
return result;
}
describe("OpenTelemetry metrics bridge", () => {
let tmpDir: tmp.DirResult;
beforeEach(() => {
tmpDir = tmp.dirSync({ unsafeCleanup: true });
});
afterEach(() => tmpDir.removeCallback());
it("snapshot is safe to call regardless of install state", () => {
expect(Array.isArray(snapshotLancedbMetrics())).toBe(true);
});
it("exports object store metrics via observable instruments", async () => {
const reader = new TestMetricReader();
const provider = new MeterProvider({ readers: [reader] });
expect(instrumentLanceDbMetrics(provider)).toBe(true);
// Generate object store activity on the local filesystem (scheme "file").
const db = await connect(tmpDir.name);
const data = Array.from({ length: 256 }, (_, i) => ({ id: i }));
const table = await db.createTable("t", data);
expect(await table.countRows()).toBe(256);
const metrics = await metricsByName(reader);
const requests = metrics.get("lance_object_store_requests_total");
expect(requests).toBeDefined();
// biome-ignore lint/suspicious/noExplicitAny: SDK point shape
const requestPoints = (requests!.dataPoints as any[]) ?? [];
expect(requestPoints.length).toBeGreaterThan(0);
for (const p of requestPoints) {
// Labelled by `operation` and `base` (the store scheme by default).
expect(p.attributes).toHaveProperty("base");
expect(p.attributes).toHaveProperty("operation");
}
const totalRequests = requestPoints.reduce((acc, p) => acc + p.value, 0);
expect(totalRequests).toBeGreaterThan(0);
// Histograms are decomposed into bucket / count / sum observable counters.
const bucket = metrics.get(
"lance_object_store_request_duration_seconds_bucket",
);
expect(bucket).toBeDefined();
// biome-ignore lint/suspicious/noExplicitAny: SDK point shape
const bucketPoints = (bucket!.dataPoints as any[]) ?? [];
expect(bucketPoints.length).toBeGreaterThan(0);
expect(bucketPoints.every((p) => "le" in p.attributes)).toBe(true);
// The implicit +Inf bucket must be present.
expect(bucketPoints.some((p) => p.attributes.le === "+Inf")).toBe(true);
const count = metrics.get(
"lance_object_store_request_duration_seconds_count",
);
expect(count).toBeDefined();
// biome-ignore lint/suspicious/noExplicitAny: SDK point shape
const countPoints = (count!.dataPoints as any[]) ?? [];
expect(countPoints.reduce((acc, p) => acc + p.value, 0)).toBeGreaterThan(0);
const sum = metrics.get("lance_object_store_request_duration_seconds_sum");
expect(sum).toBeDefined();
// biome-ignore lint/suspicious/noExplicitAny: SDK point shape
const sumPoints = (sum!.dataPoints as any[]) ?? [];
expect(sumPoints.reduce((acc, p) => acc + p.value, 0)).toBeGreaterThan(0);
// Unit handling: only `_sum` keeps the histogram's unit (seconds); `_bucket`
// and `_count` observe cumulative counts and are unitless.
expect(sum!.descriptor.unit).toBe("s");
expect(bucket!.descriptor.unit).toBe("");
expect(count!.descriptor.unit).toBe("");
await provider.shutdown();
});
});
nodejs/__test__/package.test.ts
+14
View File
@@ -0,0 +1,14 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import packageJson = require("../package.json");
describe("package metadata", () => {
it("requires Node.js type declarations compatible with the runtime", () => {
expect(packageJson.engines.node).toBe(">= 22");
expect(packageJson.peerDependencies["@types/node"]).toBe(">=22");
expect(packageJson.peerDependenciesMeta["@types/node"]).toEqual({
optional: true,
});
});
});
nodejs/__test__/permutation.test.ts
+371
View File
@@ -0,0 +1,371 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import * as tmp from "tmp";
import { Table, connect, permutationBuilder } from "../lancedb";
import { makeArrowTable } from "../lancedb/arrow";
describe("PermutationBuilder", () => {
let tmpDir: tmp.DirResult;
let table: Table;
beforeEach(async () => {
tmpDir = tmp.dirSync({ unsafeCleanup: true });
const db = await connect(tmpDir.name);
// Create test data
const data = makeArrowTable(
[
{ id: 1, value: 10 },
{ id: 2, value: 20 },
{ id: 3, value: 30 },
{ id: 4, value: 40 },
{ id: 5, value: 50 },
{ id: 6, value: 60 },
{ id: 7, value: 70 },
{ id: 8, value: 80 },
{ id: 9, value: 90 },
{ id: 10, value: 100 },
],
{ vectorColumns: {} },
);
table = await db.createTable("test_table", data);
});
afterEach(() => {
tmpDir.removeCallback();
});
test("should create permutation builder", () => {
const builder = permutationBuilder(table);
expect(builder).toBeDefined();
});
test("should execute basic permutation", async () => {
const builder = permutationBuilder(table);
const permutationTable = await builder.execute();
expect(permutationTable).toBeDefined();
const rowCount = await permutationTable.countRows();
expect(rowCount).toBe(10);
});
test("should create permutation with random splits", async () => {
const builder = permutationBuilder(table).splitRandom({
ratios: [1.0],
seed: 42,
});
const permutationTable = await builder.execute();
const rowCount = await permutationTable.countRows();
expect(rowCount).toBe(10);
});
test("should create permutation with percentage splits", async () => {
const builder = permutationBuilder(table).splitRandom({
ratios: [0.3, 0.7],
seed: 42,
});
const permutationTable = await builder.execute();
const rowCount = await permutationTable.countRows();
expect(rowCount).toBe(10);
// Check split distribution
const split0Count = await permutationTable.countRows("split_id = 0");
const split1Count = await permutationTable.countRows("split_id = 1");
expect(split0Count).toBeGreaterThan(0);
expect(split1Count).toBeGreaterThan(0);
expect(split0Count + split1Count).toBe(10);
});
test("should create permutation with count splits", async () => {
const builder = permutationBuilder(table).splitRandom({
counts: [3, 7],
seed: 42,
});
const permutationTable = await builder.execute();
const rowCount = await permutationTable.countRows();
expect(rowCount).toBe(10);
// Check split distribution
const split0Count = await permutationTable.countRows("split_id = 0");
const split1Count = await permutationTable.countRows("split_id = 1");
expect(split0Count).toBe(3);
expect(split1Count).toBe(7);
});
test("should create permutation with hash splits", async () => {
const builder = permutationBuilder(table).splitHash({
columns: ["id"],
splitWeights: [50, 50],
discardWeight: 0,
});
const permutationTable = await builder.execute();
const rowCount = await permutationTable.countRows();
expect(rowCount).toBe(10);
// Check that splits exist
const split0Count = await permutationTable.countRows("split_id = 0");
const split1Count = await permutationTable.countRows("split_id = 1");
expect(split0Count).toBeGreaterThan(0);
expect(split1Count).toBeGreaterThan(0);
expect(split0Count + split1Count).toBe(10);
});
test("should create permutation with sequential splits", async () => {
const builder = permutationBuilder(table).splitSequential({
ratios: [0.5, 0.5],
});
const permutationTable = await builder.execute();
const rowCount = await permutationTable.countRows();
expect(rowCount).toBe(10);
// Check split distribution - sequential should give exactly 5 and 5
const split0Count = await permutationTable.countRows("split_id = 0");
const split1Count = await permutationTable.countRows("split_id = 1");
expect(split0Count).toBe(5);
expect(split1Count).toBe(5);
});
test("should create permutation with calculated splits", async () => {
const builder = permutationBuilder(table).splitCalculated({
calculation: "id % 2",
});
const permutationTable = await builder.execute();
const rowCount = await permutationTable.countRows();
expect(rowCount).toBe(10);
// Check split distribution
const split0Count = await permutationTable.countRows("split_id = 0");
const split1Count = await permutationTable.countRows("split_id = 1");
expect(split0Count).toBeGreaterThan(0);
expect(split1Count).toBeGreaterThan(0);
expect(split0Count + split1Count).toBe(10);
});
test("should create permutation with shuffle", async () => {
const builder = permutationBuilder(table).shuffle({
seed: 42,
});
const permutationTable = await builder.execute();
const rowCount = await permutationTable.countRows();
expect(rowCount).toBe(10);
});
test("should create permutation with shuffle and clump size", async () => {
const builder = permutationBuilder(table).shuffle({
seed: 42,
clumpSize: 2,
});
const permutationTable = await builder.execute();
const rowCount = await permutationTable.countRows();
expect(rowCount).toBe(10);
});
test("should create permutation with filter", async () => {
const builder = permutationBuilder(table).filter("value > 50");
const permutationTable = await builder.execute();
const rowCount = await permutationTable.countRows();
expect(rowCount).toBe(5); // Values 60, 70, 80, 90, 100
});
test("should chain multiple operations", async () => {
const builder = permutationBuilder(table)
.filter("value <= 80")
.splitRandom({ ratios: [0.5, 0.5], seed: 42 })
.shuffle({ seed: 123 });
const permutationTable = await builder.execute();
const rowCount = await permutationTable.countRows();
expect(rowCount).toBe(8); // Values 10, 20, 30, 40, 50, 60, 70, 80
// Check split distribution
const split0Count = await permutationTable.countRows("split_id = 0");
const split1Count = await permutationTable.countRows("split_id = 1");
expect(split0Count).toBeGreaterThan(0);
expect(split1Count).toBeGreaterThan(0);
expect(split0Count + split1Count).toBe(8);
});
test("should throw error for invalid split arguments", () => {
const builder = permutationBuilder(table);
// Test no arguments provided
expect(() => builder.splitRandom({})).toThrow(
"Exactly one of 'ratios', 'counts', or 'fixed' must be provided",
);
// Test multiple arguments provided
expect(() =>
builder.splitRandom({ ratios: [0.5, 0.5], counts: [3, 7], seed: 42 }),
).toThrow("Exactly one of 'ratios', 'counts', or 'fixed' must be provided");
});
test("should throw error when builder is consumed", async () => {
const builder = permutationBuilder(table);
// Execute once
await builder.execute();
// Should throw error on second execution
await expect(builder.execute()).rejects.toThrow("Builder already consumed");
});
test("should accept custom split names with random splits", async () => {
const builder = permutationBuilder(table).splitRandom({
ratios: [0.3, 0.7],
seed: 42,
splitNames: ["train", "test"],
});
const permutationTable = await builder.execute();
const rowCount = await permutationTable.countRows();
expect(rowCount).toBe(10);
// Split names are provided but split_id is still numeric (0, 1, etc.)
// The names are metadata that can be used by higher-level APIs
const split0Count = await permutationTable.countRows("split_id = 0");
const split1Count = await permutationTable.countRows("split_id = 1");
expect(split0Count).toBeGreaterThan(0);
expect(split1Count).toBeGreaterThan(0);
expect(split0Count + split1Count).toBe(10);
});
test("should accept custom split names with hash splits", async () => {
const builder = permutationBuilder(table).splitHash({
columns: ["id"],
splitWeights: [50, 50],
discardWeight: 0,
splitNames: ["set_a", "set_b"],
});
const permutationTable = await builder.execute();
const rowCount = await permutationTable.countRows();
expect(rowCount).toBe(10);
// Split names are provided but split_id is still numeric
const split0Count = await permutationTable.countRows("split_id = 0");
const split1Count = await permutationTable.countRows("split_id = 1");
expect(split0Count).toBeGreaterThan(0);
expect(split1Count).toBeGreaterThan(0);
expect(split0Count + split1Count).toBe(10);
});
test("should accept custom split names with sequential splits", async () => {
const builder = permutationBuilder(table).splitSequential({
ratios: [0.5, 0.5],
splitNames: ["first", "second"],
});
const permutationTable = await builder.execute();
const rowCount = await permutationTable.countRows();
expect(rowCount).toBe(10);
// Split names are provided but split_id is still numeric
const split0Count = await permutationTable.countRows("split_id = 0");
const split1Count = await permutationTable.countRows("split_id = 1");
expect(split0Count).toBe(5);
expect(split1Count).toBe(5);
});
test("should accept custom split names with calculated splits", async () => {
const builder = permutationBuilder(table).splitCalculated({
calculation: "id % 2",
splitNames: ["even", "odd"],
});
const permutationTable = await builder.execute();
const rowCount = await permutationTable.countRows();
expect(rowCount).toBe(10);
// Split names are provided but split_id is still numeric
const split0Count = await permutationTable.countRows("split_id = 0");
const split1Count = await permutationTable.countRows("split_id = 1");
expect(split0Count).toBeGreaterThan(0);
expect(split1Count).toBeGreaterThan(0);
expect(split0Count + split1Count).toBe(10);
});
test("should persist permutation to a new table", async () => {
const db = await connect(tmpDir.name);
const builder = permutationBuilder(table)
.splitRandom({
ratios: [0.7, 0.3],
seed: 42,
splitNames: ["train", "validation"],
})
.persist(db, "my_permutation");
// Execute the builder which will persist the table
const permutationTable = await builder.execute();
// Verify the persisted table exists and can be opened
const persistedTable = await db.openTable("my_permutation");
expect(persistedTable).toBeDefined();
// Verify the persisted table has the correct number of rows
const rowCount = await persistedTable.countRows();
expect(rowCount).toBe(10);
// Verify splits exist (numeric split_id values)
const split0Count = await persistedTable.countRows("split_id = 0");
const split1Count = await persistedTable.countRows("split_id = 1");
expect(split0Count).toBeGreaterThan(0);
expect(split1Count).toBeGreaterThan(0);
expect(split0Count + split1Count).toBe(10);
// Verify the table returned by execute is the same as the persisted one
const executedRowCount = await permutationTable.countRows();
expect(executedRowCount).toBe(10);
});
test("should persist permutation with multiple operations", async () => {
const db = await connect(tmpDir.name);
const builder = permutationBuilder(table)
.filter("value > 30")
.splitRandom({ ratios: [0.5, 0.5], seed: 123, splitNames: ["a", "b"] })
.shuffle({ seed: 456 })
.persist(db, "filtered_permutation");
// Execute the builder
const permutationTable = await builder.execute();
// Verify the persisted table
const persistedTable = await db.openTable("filtered_permutation");
const rowCount = await persistedTable.countRows();
expect(rowCount).toBe(7); // Values 40, 50, 60, 70, 80, 90, 100
// Verify splits exist (numeric split_id values)
const split0Count = await persistedTable.countRows("split_id = 0");
const split1Count = await persistedTable.countRows("split_id = 1");
expect(split0Count).toBeGreaterThan(0);
expect(split1Count).toBeGreaterThan(0);
expect(split0Count + split1Count).toBe(7);
// Verify the executed table matches
const executedRowCount = await permutationTable.countRows();
expect(executedRowCount).toBe(7);
});
});
nodejs/__test__/query.test.ts
+696
View File
@@ -0,0 +1,696 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import * as tmp from "tmp";
import { type Table, connect } from "../lancedb";
import {
Field,
FixedSizeList,
Float32,
Int64,
Schema,
Utf8,
makeArrowTable,
} from "../lancedb/arrow";
import { Index } from "../lancedb/indices";
describe("Query outputSchema", () => {
let tmpDir: tmp.DirResult;
let table: Table;
beforeEach(async () => {
tmpDir = tmp.dirSync({ unsafeCleanup: true });
const db = await connect(tmpDir.name);
// Create table with explicit schema to ensure proper types
const schema = new Schema([
new Field("a", new Int64(), true),
new Field("text", new Utf8(), true),
new Field(
"vec",
new FixedSizeList(2, new Field("item", new Float32())),
true,
),
]);
const data = makeArrowTable(
[
{ a: 1n, text: "foo", vec: [1, 2] },
{ a: 2n, text: "bar", vec: [3, 4] },
{ a: 3n, text: "baz", vec: [5, 6] },
],
{ schema },
);
table = await db.createTable("test", data);
});
afterEach(() => {
tmpDir.removeCallback();
});
it("should return schema for plain query", async () => {
const schema = await table.query().outputSchema();
expect(schema.fields.length).toBe(3);
expect(schema.fields.map((f) => f.name)).toEqual(["a", "text", "vec"]);
expect(schema.fields[0].type.toString()).toBe("Int64");
expect(schema.fields[1].type.toString()).toBe("Utf8");
});
it("should return schema with dynamic projection", async () => {
const schema = await table.query().select({ bl: "a * 2" }).outputSchema();
expect(schema.fields.length).toBe(1);
expect(schema.fields[0].name).toBe("bl");
expect(schema.fields[0].type.toString()).toBe("Int64");
});
it("should return schema for vector search with _distance column", async () => {
const schema = await table
.vectorSearch([1, 2])
.select(["a"])
.outputSchema();
expect(schema.fields.length).toBe(2);
expect(schema.fields.map((f) => f.name)).toEqual(["a", "_distance"]);
expect(schema.fields[0].type.toString()).toBe("Int64");
expect(schema.fields[1].type.toString()).toBe("Float32");
});
it("should return schema for FTS search", async () => {
await table.createIndex("text", { config: Index.fts() });
const schema = await table
.search("foo", "fts")
.select(["a"])
.outputSchema();
// FTS search includes _score column in addition to selected columns
expect(schema.fields.length).toBe(2);
expect(schema.fields.map((f) => f.name)).toContain("a");
expect(schema.fields.map((f) => f.name)).toContain("_score");
const aField = schema.fields.find((f) => f.name === "a");
expect(aField?.type.toString()).toBe("Int64");
});
it.each([false, true])(
"should honor vector scoring projection (explicit score: %s)",
async (includeScore) => {
const columns = includeScore ? ["a", "_distance"] : ["a"];
const query = table
.vectorSearch([1, 2])
.select(columns)
.disableScoringAutoprojection();
const results = await query.toArrow();
expect(results.numRows).toBe(3);
expect(results.schema.fields.map((field) => field.name)).toEqual(columns);
expect(
(await query.outputSchema()).fields.map((field) => field.name),
).toEqual(columns);
},
);
it.each([false, true])(
"should honor FTS scoring projection (explicit score: %s)",
async (includeScore) => {
await table.createIndex("text", { config: Index.fts() });
const columns = includeScore ? ["a", "_score"] : ["a"];
const query = table
.query()
.fullTextSearch("foo")
.select(columns)
.disableScoringAutoprojection();
const results = await query.toArrow();
expect(results.numRows).toBe(1);
expect(results.schema.fields.map((field) => field.name)).toEqual(columns);
expect(
(await query.outputSchema()).fields.map((field) => field.name),
).toEqual(columns);
},
);
it("should preserve the scoring option when converting to a vector query", async () => {
const results = await table
.query()
.disableScoringAutoprojection()
.nearestTo([1, 2])
.select({ doubled: "a * 2" })
.toArrow();
expect(results.numRows).toBe(3);
expect(results.schema.fields.map((field) => field.name)).toEqual([
"doubled",
]);
});
it("should apply the scoring option to deferred auto queries", async () => {
await table.createIndex("text", { config: Index.fts() });
const results = await table
.search("foo")
.select(["a"])
.disableScoringAutoprojection()
.toArrow();
expect(results.numRows).toBe(1);
expect(results.schema.fields.map((field) => field.name)).toEqual(["a"]);
});
it("should return schema for take query", async () => {
const schema = await table.takeOffsets([0]).select(["text"]).outputSchema();
expect(schema.fields.length).toBe(1);
expect(schema.fields[0].name).toBe("text");
expect(schema.fields[0].type.toString()).toBe("Utf8");
});
it("should return full schema when no select is specified", async () => {
const schema = await table.query().outputSchema();
// Should return all columns
expect(schema.fields.length).toBe(3);
});
});
describe("Query empty results", () => {
let tmpDir: tmp.DirResult;
let table: Table;
beforeEach(async () => {
tmpDir = tmp.dirSync({ unsafeCleanup: true });
const db = await connect(tmpDir.name);
const schema = new Schema(
[
new Field("id", new Int64(), false),
new Field(
"text",
new Utf8(),
true,
new Map([["description", "searchable text"]]),
),
new Field(
"vector",
new FixedSizeList(2, new Field("item", new Float32())),
true,
),
],
new Map([["source", "empty-query-regression"]]),
);
table = await db.createTable(
"test",
makeArrowTable([{ id: 1n, text: "hello", vector: [1, 2] }], { schema }),
);
});
afterEach(() => {
tmpDir.removeCallback();
});
it.each([
{
name: "filtered scan",
query: (table: Table) => table.query().where("id < 0"),
fields: ["id", "text", "vector"],
},
{
name: "zero limit",
query: (table: Table) => table.query().limit(0),
fields: ["id", "text", "vector"],
},
{
name: "column projection",
query: (table: Table) =>
table.query().where("id < 0").select(["text", "id"]),
fields: ["text", "id"],
},
{
name: "dynamic projection",
query: (table: Table) =>
table.query().where("id < 0").select({ doubled: "id * 2" }),
fields: ["doubled"],
},
{
name: "filtered vector search",
query: (table: Table) =>
table.search([0.5, 0.5]).where("id < 0").limit(3),
fields: ["id", "text", "vector", "_distance"],
},
{
name: "vector projection with row id",
query: (table: Table) =>
table
.vectorSearch([0.5, 0.5])
.where("id < 0")
.select(["id"])
.withRowId(),
fields: ["id", "_distance", "_rowid"],
},
{
name: "fast search without an index",
query: (table: Table) => table.search([0.5, 0.5]).fastSearch(),
fields: ["id", "text", "vector", "_distance"],
},
{
name: "take with no offsets",
query: (table: Table) => table.takeOffsets([]).select(["text"]),
fields: ["text"],
},
])("preserves the output schema for $name", async ({ query, fields }) => {
const result = await query(table).toArrow({ maxBatchLength: 1 });
expect(result.numRows).toBe(0);
expect(result.schema.fields.map((field) => field.name)).toEqual(fields);
for (const name of fields) {
expect(result.getChild(name)).not.toBeNull();
expect(result.getChild(name)?.length).toBe(0);
}
});
it("preserves types, nullability, and metadata", async () => {
const result = await table.query().where("id < 0").toArrow();
const schema = await table.schema();
expect(result.schema).toEqual(schema);
expect(result.getChild("id")?.type.toString()).toBe("Int64");
expect(result.schema.fields[0].nullable).toBe(false);
expect(result.schema.fields[1].metadata.get("description")).toBe(
"searchable text",
);
expect(result.schema.metadata.get("source")).toBe("empty-query-regression");
});
it("preserves the score column for full-text search with no matches", async () => {
await table.createIndex("text", { config: Index.fts() });
const result = await table
.search("missing", "fts")
.select(["id"])
.toArrow();
expect(result.numRows).toBe(0);
expect(result.schema.fields.map((field) => field.name)).toEqual([
"id",
"_score",
]);
expect(result.getChild("_score")?.type.toString()).toBe("Float32");
});
});
describe("Search pagination", () => {
let tmpDir: tmp.DirResult;
let table: Table;
beforeEach(async () => {
tmpDir = tmp.dirSync({ unsafeCleanup: true });
const db = await connect(tmpDir.name);
const schema = new Schema([
new Field("id", new Int64(), false),
new Field("text", new Utf8(), false),
new Field(
"vector",
new FixedSizeList(2, new Field("item", new Float32())),
false,
),
]);
const data = makeArrowTable(
[
{ id: 1n, text: "common", vector: [0, 0] },
{ id: 2n, text: "common common", vector: [1, 1] },
{ id: 3n, text: "common common common", vector: [2, 2] },
{ id: 4n, text: "common common common common", vector: [3, 3] },
],
{ schema },
);
table = await db.createTable("test", data);
});
afterEach(() => {
tmpDir.removeCallback();
});
it("applies offset after the vector search limit", async () => {
const allResults = await table
.vectorSearch([0, 0])
.select(["id"])
.limit(4)
.toArray();
const secondPage = await table
.vectorSearch([0, 0])
.select(["id"])
.limit(2)
.offset(2)
.toArray();
expect(allResults).toHaveLength(4);
expect(secondPage).toHaveLength(2);
expect(secondPage.map((row) => row.id)).toEqual(
allResults.slice(2, 4).map((row) => row.id),
);
});
it("applies offset after the full-text search limit", async () => {
await table.createIndex("text", { config: Index.fts() });
const allResults = await table
.search("common", "fts")
.select(["id"])
.limit(4)
.toArray();
const secondPage = await table
.search("common", "fts")
.select(["id"])
.limit(2)
.offset(2)
.toArray();
expect(allResults).toHaveLength(4);
expect(secondPage).toHaveLength(2);
expect(secondPage.map((row) => row.id)).toEqual(
allResults.slice(2, 4).map((row) => row.id),
);
});
});
describe("Query orderBy", () => {
let tmpDir: tmp.DirResult;
let table: Table;
beforeEach(async () => {
tmpDir = tmp.dirSync({ unsafeCleanup: true });
const db = await connect(tmpDir.name);
// Create table with numeric data for sorting
const schema = new Schema([
new Field("id", new Int64(), true),
new Field("score", new Float32(), true),
new Field("name", new Utf8(), true),
]);
const data = makeArrowTable(
[
{ id: 1n, score: 3.5, name: "charlie" },
{ id: 2n, score: 1.2, name: "alice" },
{ id: 3n, score: 2.8, name: "bob" },
{ id: 4n, score: 0.5, name: "david" },
{ id: 5n, score: 4.1, name: "eve" },
],
{ schema },
);
table = await db.createTable("test", data);
});
afterEach(() => {
tmpDir.removeCallback();
});
it("should sort by single column ascending", async () => {
const results = await table
.query()
.orderBy({ columnName: "score", ascending: true, nullsFirst: false })
.toArray();
expect(results.length).toBe(5);
// Verify ascending order
expect(results[0].score).toBeCloseTo(0.5, 0.001);
expect(results[1].score).toBeCloseTo(1.2, 0.001);
expect(results[2].score).toBeCloseTo(2.8, 0.001);
expect(results[3].score).toBeCloseTo(3.5, 0.001);
expect(results[4].score).toBeCloseTo(4.1, 0.001);
});
it("should sort by single column descending", async () => {
const results = await table
.query()
.orderBy({ columnName: "score", ascending: false, nullsFirst: false })
.toArray();
expect(results.length).toBe(5);
// Verify descending order
expect(results[0].score).toBeCloseTo(4.1, 0.001);
expect(results[1].score).toBeCloseTo(3.5, 0.001);
expect(results[2].score).toBeCloseTo(2.8, 0.001);
expect(results[3].score).toBeCloseTo(1.2, 0.001);
expect(results[4].score).toBeCloseTo(0.5, 0.001);
});
it("should use ascending as default direction", async () => {
const results = await table
.query()
.orderBy({ columnName: "score" })
.toArray();
expect(results.length).toBe(5);
// Verify ascending order (default)
expect(results[0].score).toBeCloseTo(0.5, 0.001);
expect(results[1].score).toBeCloseTo(1.2, 0.001);
expect(results[2].score).toBeCloseTo(2.8, 0.001);
expect(results[3].score).toBeCloseTo(3.5, 0.001);
expect(results[4].score).toBeCloseTo(4.1, 0.001);
});
it("should sort by string column", async () => {
const results = await table
.query()
.orderBy({ columnName: "name" })
.toArray();
expect(results.length).toBe(5);
// Verify alphabetical order
expect(results[0].name).toBe("alice");
expect(results[1].name).toBe("bob");
expect(results[2].name).toBe("charlie");
expect(results[3].name).toBe("david");
expect(results[4].name).toBe("eve");
});
it("should support method chaining with where", async () => {
const results = await table
.query()
.where("score > 2.0")
.orderBy({ columnName: "score" })
.toArray();
expect(results.length).toBe(3);
// Verify filtered and sorted
expect(results[0].score).toBeCloseTo(2.8, 0.001);
expect(results[1].score).toBeCloseTo(3.5, 0.001);
expect(results[2].score).toBeCloseTo(4.1, 0.001);
});
it("should combine repeated where clauses with AND", async () => {
const results = await table
.query()
.where("score > 1.0")
.where("score < 3.0")
.orderBy({ columnName: "score" })
.toArray();
// Only rows matching both predicates should be returned, rather than the
// second where() silently replacing the first.
expect(results.length).toBe(2);
expect(results[0].score).toBeCloseTo(1.2, 0.001);
expect(results[1].score).toBeCloseTo(2.8, 0.001);
});
it("should support method chaining with limit", async () => {
const results = await table
.query()
.orderBy({ columnName: "score", ascending: false })
.limit(3)
.toArray();
expect(results.length).toBe(3);
// Verify top 3 in descending order
expect(results[0].score).toBeCloseTo(4.1, 0.001);
expect(results[1].score).toBeCloseTo(3.5, 0.001);
expect(results[2].score).toBeCloseTo(2.8, 0.001);
});
it("should support method chaining with offset", async () => {
const results = await table
.query()
.orderBy({ columnName: "score" })
.offset(2)
.limit(2)
.toArray();
expect(results.length).toBe(2);
// Verify results skip first 2 and take next 2
expect(results[0].score).toBeCloseTo(2.8, 0.001);
expect(results[1].score).toBeCloseTo(3.5, 0.001);
});
it("should support method chaining with select", async () => {
const results = await table
.query()
.orderBy({ columnName: "name" })
.select(["name", "score"])
.toArray();
expect(results.length).toBe(5);
// Verify only selected columns are present
expect(Object.keys(results[0])).toEqual(["name", "score"]);
expect(Object.keys(results[4])).toEqual(["name", "score"]);
// Verify sorted by name
expect(results[0].name).toBe("alice");
expect(results[4].name).toBe("eve");
});
it("should support complex method chaining", async () => {
const results = await table
.query()
.where("score > 1.0")
.orderBy({ columnName: "score", ascending: false })
.limit(3)
.select(["id", "score", "name"])
.toArray();
expect(results.length).toBe(3);
// Verify filtered, sorted, limited, and projected
expect(results[0].score).toBeCloseTo(4.1, 0.001);
expect(results[1].score).toBeCloseTo(3.5, 0.001);
expect(results[2].score).toBeCloseTo(2.8, 0.001);
expect(Object.keys(results[0])).toEqual(["id", "score", "name"]);
});
it("should support multi-column ordering and null placement", async () => {
const schema = new Schema([
new Field("group", new Int64(), true),
new Field("score", new Float32(), true),
new Field("name", new Utf8(), true),
]);
const data = makeArrowTable(
[
{ group: 1n, score: null, name: "z" },
{ group: 1n, score: 1.0, name: "b" },
{ group: 1n, score: 1.0, name: "a" },
{ group: 2n, score: 0.5, name: "c" },
],
{ schema },
);
const nullTable = await (await connect(tmpDir.name)).createTable(
"test_multi_order",
data,
{ mode: "overwrite" },
);
const results = await nullTable
.query()
.orderBy([
{ columnName: "group", ascending: true, nullsFirst: false },
{ columnName: "score", ascending: true, nullsFirst: true },
{ columnName: "name", ascending: true, nullsFirst: false },
])
.toArray();
expect(results.map((r) => [r.group, r.score, r.name])).toEqual([
[1n, null, "z"],
[1n, 1.0, "a"],
[1n, 1.0, "b"],
[2n, 0.5, "c"],
]);
});
});
describe("IVF search effort", () => {
let directory: tmp.DirResult;
let table: Table;
const vector = Array.from({ length: 16 }, (_, i) => (i === 0 ? 1 : 0));
beforeEach(async () => {
directory = tmp.dirSync({ unsafeCleanup: true });
const db = await connect(directory.name);
const rows = Array.from({ length: 512 }, (_, id) => ({
id,
vector: Array.from(
{ length: 16 },
(_, j) =>
(Math.floor(id / 32) === j ? 1 : 0) + Math.sin(id * 17 + j) * 0.001,
),
}));
table = await db.createTable("effort", rows.slice(0, 256));
await table.add(rows.slice(256));
await table.createIndex("vector", {
config: Index.ivfFlat({ numPartitions: 16 }),
});
});
afterEach(() => {
directory.removeCallback();
});
it.each([
[null, 6],
[0, 1],
[0.25, 3],
[0.5, 6],
[0.75, 10],
[1, 16],
])(
"preserves effort %s through execution and probe precedence",
async (effort, expected) => {
const partitions = async (query: ReturnType<Table["vectorSearch"]>) => {
const plan = await query.limit(8).analyzePlan();
const matches = [...plan.matchAll(/partitions_searched=(\d+)/g)];
expect(matches.length).toBeGreaterThan(0);
return matches.reduce((total, match) => total + Number(match[1]), 0);
};
expect(
await partitions(table.vectorSearch(vector).searchEffort(effort)),
).toBe(expected);
expect(
await partitions(
table.vectorSearch(vector).nprobes(1).searchEffort(effort),
),
).toBe(effort === null ? 1 : expected);
expect(
await partitions(
table.vectorSearch(vector).searchEffort(effort).nprobes(1),
),
).toBe(effort === null ? 1 : expected);
expect(
await partitions(
table
.vectorSearch(vector)
.searchEffort(effort)
.nprobes(1)
.searchEffort(null),
),
).toBe(1);
expect(
await partitions(
table.vectorSearch(vector).searchEffort(effort).maximumNprobes(2),
),
).toBeLessThanOrEqual(2);
const actual = await table
.vectorSearch(vector)
.searchEffort(effort)
.limit(8)
.toArray();
const exact = await table
.vectorSearch(vector)
.bypassVectorIndex()
.limit(8)
.toArray();
const ids = new Set(exact.map((row) => row.id));
expect(
actual.filter((row) => ids.has(row.id)).length / 8,
).toBeGreaterThanOrEqual(0.5);
},
);
it.each([
-0.1,
1.1,
Number.NaN,
Number.POSITIVE_INFINITY,
Number.NEGATIVE_INFINITY,
])("rejects invalid effort %s before execution", (effort) => {
expect(() => table.vectorSearch(vector).searchEffort(effort)).toThrow(
"search_effort must be finite and in [0, 1]",
);
expect(() =>
table.vectorSearch(vector).fullTextSearch("dog").searchEffort(effort),
).toThrow("search_effort must be finite and in [0, 1]");
});
});
nodejs/__test__/registry.test.ts
+76 -8
View File
@@ -3,13 +3,10 @@
import * as apiArrow from "apache-arrow";
import * as arrow15 from "apache-arrow-15";
import * as arrow16 from "apache-arrow-16";
import * as arrow17 from "apache-arrow-17";
import * as arrow18 from "apache-arrow-18";
import * as tmp from "tmp";
import { arrowVersions } from "./arrow_versions";
import { connect } from "../lancedb";
import {
EmbeddingFunction,
@@ -18,7 +15,7 @@ import {
} from "../lancedb/embedding";
import { getRegistry, register } from "../lancedb/embedding/registry";
describe.each([arrow15, arrow16, arrow17, arrow18])("LanceSchema", (arrow) => {
describe.each(arrowVersions)("LanceSchema", (arrow) => {
test("should preserve input order", async () => {
const schema = LanceSchema({
id: new arrow.Int32(),
@@ -29,7 +26,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])("LanceSchema", (arrow) => {
});
});
describe.each([arrow15, arrow16, arrow17, arrow18])("Registry", (arrow) => {
describe.each(arrowVersions)("Registry", (arrow) => {
let tmpDir: tmp.DirResult;
beforeEach(() => {
tmpDir = tmp.dirSync({ unsafeCleanup: true });
@@ -106,6 +103,77 @@ describe.each([arrow15, arrow16, arrow17, arrow18])("Registry", (arrow) => {
'Embedding function with alias "mock-embedding" already exists',
);
});
test("parseFunctions keeps entries sharing a function name", async () => {
class MockEmbeddingFunction extends EmbeddingFunction<string> {
ndims() {
return 3;
}
embeddingDataType() {
return new arrow.Float32() as apiArrow.Float;
}
async computeSourceEmbeddings(data: string[]) {
return data.map(() => [1, 2, 3]);
}
}
register("mock-embedding")(MockEmbeddingFunction);
const parsed = await getRegistry().parseFunctions(
new Map([
[
"embedding_functions",
JSON.stringify([
{
name: "mock-embedding",
sourceColumn: "text",
vectorColumn: "vector_a",
model: {},
},
{
name: "mock-embedding",
sourceColumn: "text",
vectorColumn: "vector_b",
model: {},
},
]),
],
]),
);
expect([...parsed.values()].map((f) => f.vectorColumn)).toEqual([
"vector_a",
"vector_b",
]);
// The Python bindings write snake_case keys.
const snake = await getRegistry().parseFunctions(
new Map([
[
"embedding_functions",
JSON.stringify([
{
name: "mock-embedding",
// biome-ignore lint/style/useNamingConvention: the Python wire spelling
source_column: "text",
// biome-ignore lint/style/useNamingConvention: the Python wire spelling
vector_column: "vector_a",
model: {},
},
{
name: "mock-embedding",
// biome-ignore lint/style/useNamingConvention: the Python wire spelling
source_column: "text",
// biome-ignore lint/style/useNamingConvention: the Python wire spelling
vector_column: "vector_b",
model: {},
},
]),
],
]),
);
expect([...snake.keys()]).toEqual(["vector_a", "vector_b"]);
expect([...snake.values()].map((f) => f.sourceColumn)).toEqual([
"text",
"text",
]);
});
test("schema should contain correct metadata", async () => {
class MockEmbeddingFunction extends EmbeddingFunction<string> {
constructor(args: FunctionOptions = {}) {
@@ -169,7 +237,7 @@ describe("Registry.setVar", () => {
return data.map(() => [1, 2, 3]);
}
embeddingDataType() {
return new arrow18.Float32() as apiArrow.Float;
return new apiArrow.Float32();
}
protected getSensitiveKeys() {
return ["secretKey"];
nodejs/__test__/remote.test.ts
+1719 -3
View File
File diff suppressed because it is too large Load Diff
nodejs/__test__/rerankers.test.ts
+89
View File
@@ -1,6 +1,8 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { spawn } from "node:child_process";
import * as path from "node:path";
import { RecordBatch } from "apache-arrow";
import * as tmp from "tmp";
import { Connection, Index, Table, connect, makeArrowTable } from "../lancedb";
@@ -76,4 +78,91 @@ describe("rerankers", function () {
expect(result).toHaveLength(2);
});
it("does not keep process alive after rerank query", async function () {
const script = `
import * as lancedb from "./dist/index.js";
import * as os from "node:os";
import * as path from "node:path";
import * as fs from "node:fs/promises";
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "lancedb-rerank-exit-"));
const db = await lancedb.connect(dir);
const table = await db.createTable("test", [{ text: "hello", vector: [1, 2, 3] }], {
mode: "overwrite",
});
await table.createIndex("text", { config: lancedb.Index.fts() });
await table.waitForIndex(["text_idx"], 30);
const reranker = await lancedb.rerankers.RRFReranker.create();
await table
.query()
.nearestTo([1, 2, 3])
.fullTextSearch("hello")
.rerank(reranker)
.toArray();
table.close();
db.close();
`;
await new Promise<void>((resolve, reject) => {
const child = spawn(
process.execPath,
["--input-type=module", "-e", script],
{
cwd: path.resolve(__dirname, ".."),
stdio: ["ignore", "pipe", "pipe"],
},
);
let stdout = "";
let stderr = "";
child.stdout.on("data", (chunk) => {
stdout += chunk.toString();
});
child.stderr.on("data", (chunk) => {
stderr += chunk.toString();
});
const timeout = setTimeout(() => {
child.kill();
reject(
new Error(
`child process did not exit in time\nstdout:\n${stdout}\nstderr:\n${stderr}`,
),
);
}, 20_000);
child.on("error", (err) => {
clearTimeout(timeout);
reject(err);
});
child.on("exit", (code, signal) => {
clearTimeout(timeout);
if (signal !== null) {
reject(
new Error(
`child process exited with signal ${signal}\nstdout:\n${stdout}\nstderr:\n${stderr}`,
),
);
return;
}
if (code !== 0) {
reject(
new Error(
`child process exited with code ${code}\nstdout:\n${stdout}\nstderr:\n${stderr}`,
),
);
return;
}
resolve();
});
});
});
});
nodejs/__test__/sanitize.test.ts
+307
View File
@@ -0,0 +1,307 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import * as arrow from "../lancedb/arrow";
import {
sanitizeField,
sanitizeMap,
sanitizeSchema,
sanitizeType,
} from "../lancedb/sanitize";
import { arrow21 } from "./arrow_versions";
describe("sanitize", function () {
describe("sanitizeType function", function () {
it("should handle type objects", function () {
const type = new arrow.Int32();
const result = sanitizeType(type);
expect(result.typeId).toBe(arrow.Type.Int);
expect((result as arrow.Int).bitWidth).toBe(32);
expect((result as arrow.Int).isSigned).toBe(true);
const floatType = {
typeId: 3, // Type.Float = 3
precision: 2,
toString: () => "Float",
isFloat: true,
isFixedWidth: true,
};
const floatResult = sanitizeType(floatType);
expect(floatResult).toBeInstanceOf(arrow.DataType);
expect(floatResult.typeId).toBe(arrow.Type.Float);
const floatResult2 = sanitizeType({ ...floatType, typeId: () => 3 });
expect(floatResult2).toBeInstanceOf(arrow.DataType);
expect(floatResult2.typeId).toBe(arrow.Type.Float);
});
const allTypeNameTestCases = [
["null", new arrow.Null()],
["binary", new arrow.Binary()],
["utf8", new arrow.Utf8()],
["bool", new arrow.Bool()],
["int8", new arrow.Int8()],
["int16", new arrow.Int16()],
["int32", new arrow.Int32()],
["int64", new arrow.Int64()],
["uint8", new arrow.Uint8()],
["uint16", new arrow.Uint16()],
["uint32", new arrow.Uint32()],
["uint64", new arrow.Uint64()],
["float16", new arrow.Float16()],
["float32", new arrow.Float32()],
["float64", new arrow.Float64()],
["datemillisecond", new arrow.DateMillisecond()],
["dateday", new arrow.DateDay()],
["timenanosecond", new arrow.TimeNanosecond()],
["timemicrosecond", new arrow.TimeMicrosecond()],
["timemillisecond", new arrow.TimeMillisecond()],
["timesecond", new arrow.TimeSecond()],
["intervaldaytime", new arrow.IntervalDayTime()],
["intervalyearmonth", new arrow.IntervalYearMonth()],
["durationnanosecond", new arrow.DurationNanosecond()],
["durationmicrosecond", new arrow.DurationMicrosecond()],
["durationmillisecond", new arrow.DurationMillisecond()],
["durationsecond", new arrow.DurationSecond()],
["largebinary", new arrow.LargeBinary()],
["largeutf8", new arrow.LargeUtf8()],
] as const;
it.each(allTypeNameTestCases)(
'should map type name "%s" to %s',
function (name, expected) {
const result = sanitizeType(name);
expect(result).toBeInstanceOf(expected.constructor);
},
);
const caseVariationTestCases = [
["NULL", new arrow.Null()],
["Utf8", new arrow.Utf8()],
["FLOAT32", new arrow.Float32()],
["DaTedAy", new arrow.DateDay()],
] as const;
it.each(caseVariationTestCases)(
'should be case insensitive for type name "%s" mapped to %s',
function (name, expected) {
const result = sanitizeType(name);
expect(result).toBeInstanceOf(expected.constructor);
},
);
it("should throw error for unrecognized type name", function () {
expect(() => sanitizeType("invalid_type")).toThrow(
"Unrecognized type name in schema: invalid_type",
);
});
// These types only exist in Arrow 20 (IntervalMonthDayNano) and 21 (the
// rest), so the objects come from a second copy of the newest release. The
// library itself may be running on an older Arrow (see the CI matrix), in
// which case sanitizing them has to fail with a clear message instead.
const hostArrow = arrow as unknown as Record<string, unknown>;
const hostSupportsNewerTypes = typeof hostArrow.Utf8View === "function";
const whenSupported = hostSupportsNewerTypes ? describe : describe.skip;
const whenUnsupported = hostSupportsNewerTypes ? describe.skip : describe;
const newerTypeTestCases = [
["Utf8View", () => new arrow21.Utf8View()],
["BinaryView", () => new arrow21.BinaryView()],
["IntervalMonthDayNano", () => new arrow21.IntervalMonthDayNano()],
[
"LargeList",
() =>
new arrow21.LargeList(new arrow21.Field("item", new arrow21.Utf8())),
],
] as const;
whenSupported("types added in Arrow 20 and 21", function () {
it.each(newerTypeTestCases)(
"should sanitize a %s type from another copy of Arrow",
function (name, makeForeign) {
const foreign = makeForeign();
const result = sanitizeType(foreign);
// Interval subclasses share `Type.Interval`, so the sanitized type
// is the generic `Interval` carrying the same unit, as for the
// other units.
const expected = name === "IntervalMonthDayNano" ? "Interval" : name;
expect(result).toBeInstanceOf(hostArrow[expected] as () => unknown);
expect(result.typeId).toBe(foreign.typeId);
expect(result.toString()).toBe(foreign.toString());
},
);
it.each([
["binaryview", "BinaryView"],
["utf8view", "Utf8View"],
["intervalmonthdaynano", "Interval"],
])('should map type name "%s" to %s', function (name, expected) {
expect(sanitizeType(name)).toBeInstanceOf(
hostArrow[expected] as () => unknown,
);
});
it("should sanitize the child field of a LargeList", function () {
const foreign = new arrow21.LargeList(
new arrow21.Field("item", new arrow21.Int32(), false),
);
const result = sanitizeType(foreign) as arrow.List;
expect(result.children[0]).toBeInstanceOf(arrow.Field);
expect(result.children[0].name).toBe("item");
expect(result.children[0].nullable).toBe(false);
const child = result.children[0].type;
expect(arrow.DataType.isInt(child)).toBe(true);
expect((child as arrow.Int).bitWidth).toBe(32);
expect((child as arrow.Int).isSigned).toBe(true);
});
});
whenUnsupported("types newer than the installed Arrow", function () {
it.each(
newerTypeTestCases.filter(([name]) => name !== "IntervalMonthDayNano"),
)(
"should explain that %s needs a newer apache-arrow",
function (name, makeForeign) {
expect(() => sanitizeType(makeForeign())).toThrow(
`The Arrow type ${name} requires apache-arrow`,
);
},
);
it("should sanitize IntervalMonthDayNano to a generic Interval", () => {
// The subclass shares `Type.Interval`, which every supported Arrow
// release has; only the unit is new.
const result = sanitizeType(new arrow21.IntervalMonthDayNano());
expect(result).toBeInstanceOf(arrow.Interval);
expect((result as arrow.Interval).unit).toBe(
arrow21.IntervalUnit.MONTH_DAY_NANO,
);
});
});
it("should rebuild a schema from a second copy of the same Arrow version", function () {
// Arrow 21 makes `instanceof Schema` succeed across copies of the library
// but not `instanceof <DataType>`; the schema must still be rebuilt.
const foreign = new arrow21.Schema([
new arrow21.Field("x", new arrow21.Float32(), true),
]);
// Only Arrow 21+ overrides `Symbol.hasInstance` this way.
const crossCopyInstanceof = Object.getOwnPropertySymbols(
arrow.Schema,
).includes(Symbol.hasInstance);
expect(foreign instanceof arrow.Schema).toBe(crossCopyInstanceof);
const result = sanitizeSchema(foreign);
expect(result).not.toBe(foreign);
const type = result.fields[0].type;
expect(type).toBeInstanceOf(arrow.DataType);
expect(arrow.DataType.isFloat(type)).toBe(true);
expect((type as arrow.Float).precision).toBe(arrow.Precision.SINGLE);
expect(() => arrow.makeBuilder({ type })).not.toThrow();
});
});
describe("sanitizeField function", function () {
it("should handle field with string type name", function () {
const field = sanitizeField({
name: "string_field",
type: "utf8",
nullable: true,
metadata: new Map([["key", "value"]]),
});
expect(field).toBeInstanceOf(arrow.Field);
expect(field.name).toBe("string_field");
expect(field.type).toBeInstanceOf(arrow.Utf8);
expect(field.nullable).toBe(true);
expect(field.metadata?.get("key")).toBe("value");
});
it("should handle field with type object", function () {
const floatType = {
typeId: 3, // Float
precision: 32,
};
const field = sanitizeField({
name: "float_field",
type: floatType,
nullable: false,
});
expect(field).toBeInstanceOf(arrow.Field);
expect(field.name).toBe("float_field");
expect(field.type).toBeInstanceOf(arrow.DataType);
expect(field.type.typeId).toBe(arrow.Type.Float);
expect((field.type as arrow.Float64).precision).toBe(32);
expect(field.nullable).toBe(false);
});
it("should handle field with direct Type instance", function () {
const field = sanitizeField({
name: "bool_field",
type: new arrow.Bool(),
nullable: true,
});
expect(field).toBeInstanceOf(arrow.Field);
expect(field.name).toBe("bool_field");
expect(field.type).toBeInstanceOf(arrow.Bool);
expect(field.nullable).toBe(true);
});
it("should throw error for invalid field object", function () {
expect(() =>
sanitizeField({
type: "int32",
nullable: true,
}),
).toThrow(
"The field passed in is missing a `type`/`name`/`nullable` property",
);
// Invalid type
expect(() =>
sanitizeField({
name: "invalid",
type: { invalid: true },
nullable: true,
}),
).toThrow("Expected a Type to have a typeId property");
// Invalid nullable
expect(() =>
sanitizeField({
name: "invalid_nullable",
type: "int32",
nullable: "not a boolean",
}),
).toThrow("The field passed in had a non-boolean `nullable` property");
});
it("should report error for invalid type name", function () {
expect(() =>
sanitizeField({
name: "invalid_field",
type: "invalid_type",
nullable: true,
}),
).toThrow(
"Unable to sanitize type for field: invalid_field due to error: Error: Unrecognized type name in schema: invalid_type",
);
});
});
describe("sanitizeMap function", function () {
it.each([
["no children", []],
["two children", [{}, {}]],
])("should reject a Map type with %s", function (_, children) {
expect(() => sanitizeMap({ children, keysSorted: false })).toThrow(
"Expected a Map type to have exactly one child",
);
});
});
});
nodejs/__test__/scannable.test.ts
+438
View File
@@ -0,0 +1,438 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import {
Field,
Float16,
Int32,
type RecordBatch,
RecordBatchReader,
Schema,
tableToIPC,
} from "apache-arrow";
import { makeArrowTable, makeEmptyTable } from "../lancedb/arrow";
import { Scannable } from "../lancedb/scannable";
function makeTable() {
return makeArrowTable(
[
{ id: 1, name: "a" },
{ id: 2, name: "b" },
{ id: 3, name: "c" },
],
{ vectorColumns: {} },
);
}
async function makeReader(): Promise<RecordBatchReader> {
// `RecordBatchReader.from()` returns an unopened reader; `.schema` is only
// populated after `.open()`. Opening sync readers is synchronous.
const reader = RecordBatchReader.from(tableToIPC(makeTable()));
return reader.open() as RecordBatchReader;
}
describe("Scannable", () => {
describe("fromTable", () => {
test("reflects schema, numRows, and defaults rescannable=true", async () => {
const table = makeTable();
const scannable = await Scannable.fromTable(table);
expect(scannable.schema).toBe(table.schema);
expect(scannable.numRows).toBe(table.numRows);
expect(scannable.rescannable).toBe(true);
});
test("throws when opts.numRows does not match table.numRows", async () => {
await expect(
Scannable.fromTable(makeTable(), { numRows: 42 }),
).rejects.toThrow(/does not match table\.numRows/);
});
test("throws when opts.rescannable is false", async () => {
await expect(
Scannable.fromTable(makeTable(), { rescannable: false }),
).rejects.toThrow(/always rescannable/);
});
});
describe("fromRecordBatchReader", () => {
test("reflects schema and defaults numRows=null, rescannable=false", async () => {
const reader = await makeReader();
const scannable = await Scannable.fromRecordBatchReader(reader);
expect(scannable.schema).toBe(reader.schema);
expect(scannable.numRows).toBeNull();
expect(scannable.rescannable).toBe(false);
});
test("honors numRows override", async () => {
const scannable = await Scannable.fromRecordBatchReader(
await makeReader(),
{ numRows: 3 },
);
expect(scannable.numRows).toBe(3);
expect(scannable.rescannable).toBe(false);
});
test("rescannable: false explicit does not throw", async () => {
const reader = await makeReader();
const scannable = await Scannable.fromRecordBatchReader(reader, {
rescannable: false,
});
expect(scannable.rescannable).toBe(false);
});
test("throws when opts.rescannable is true", async () => {
const reader = await makeReader();
await expect(
Scannable.fromRecordBatchReader(reader, { rescannable: true }),
).rejects.toThrow(/does not accept rescannable/);
});
test("throws when opts.rescannable is true even alongside numRows", async () => {
const reader = await makeReader();
await expect(
Scannable.fromRecordBatchReader(reader, {
numRows: 3,
rescannable: true,
}),
).rejects.toThrow(/does not accept rescannable/);
});
});
describe("fromIterable", () => {
test("accepts a sync iterable of batches", async () => {
const table = makeTable();
const scannable = await Scannable.fromIterable(
table.schema,
table.batches,
);
expect(scannable.schema).toBe(table.schema);
expect(scannable.numRows).toBeNull();
expect(scannable.rescannable).toBe(false);
});
test("accepts an async iterable of batches", async () => {
const table = makeTable();
async function* generator(): AsyncGenerator<RecordBatch> {
for (const batch of table.batches) {
yield batch;
}
}
const scannable = await Scannable.fromIterable(table.schema, generator());
expect(scannable.schema).toBe(table.schema);
expect(scannable.rescannable).toBe(false);
});
describe("rescannable: true detection", () => {
// Replayable inputs: [Symbol.iterator]() / [Symbol.asyncIterator]()
// returns a fresh iterator each call. Must NOT throw.
test("Array passes (fresh ArrayIterator each call)", async () => {
const table = makeTable();
const scannable = await Scannable.fromIterable(
table.schema,
table.batches,
{ rescannable: true },
);
expect(scannable.rescannable).toBe(true);
});
test("Set passes (fresh SetIterator each call)", async () => {
const table = makeTable();
const set = new Set<RecordBatch>(table.batches);
const scannable = await Scannable.fromIterable(table.schema, set, {
rescannable: true,
});
expect(scannable.rescannable).toBe(true);
});
test("custom Iterable returning a fresh iterator passes", async () => {
const table = makeTable();
const replayable: Iterable<RecordBatch> = {
[Symbol.iterator]() {
return table.batches[Symbol.iterator]();
},
};
const scannable = await Scannable.fromIterable(
table.schema,
replayable,
{ rescannable: true },
);
expect(scannable.rescannable).toBe(true);
});
test("object with generator method passes (fresh generator each call)", async () => {
const table = makeTable();
const replayable: Iterable<RecordBatch> = {
*[Symbol.iterator]() {
for (const batch of table.batches) yield batch;
},
};
const scannable = await Scannable.fromIterable(
table.schema,
replayable,
{ rescannable: true },
);
expect(scannable.rescannable).toBe(true);
});
test("empty Array passes (replayable degenerate case)", async () => {
const schema = makeTable().schema;
const scannable = await Scannable.fromIterable(
schema,
[] as RecordBatch[],
{ rescannable: true },
);
expect(scannable.rescannable).toBe(true);
});
// One-shot inputs: [Symbol.iterator]() / [Symbol.asyncIterator]()
// returns the same object, or the input is already-an-iterator.
// Must throw with a /one-shot/ message.
test("sync generator throws", async () => {
const table = makeTable();
function* generator(): Generator<RecordBatch> {
for (const batch of table.batches) yield batch;
}
await expect(
Scannable.fromIterable(table.schema, generator(), {
rescannable: true,
}),
).rejects.toThrow(/one-shot/);
});
test("async generator throws", async () => {
const table = makeTable();
async function* generator(): AsyncGenerator<RecordBatch> {
for (const batch of table.batches) yield batch;
}
await expect(
Scannable.fromIterable(table.schema, generator(), {
rescannable: true,
}),
).rejects.toThrow(/one-shot/);
});
test("empty generator throws (one-shot degenerate case)", async () => {
const schema = makeTable().schema;
function* generator(): Generator<RecordBatch> {
// intentionally empty; yields nothing.
}
await expect(
Scannable.fromIterable(schema, generator(), { rescannable: true }),
).rejects.toThrow(/one-shot/);
});
test("custom self-iterator throws", async () => {
const table = makeTable();
const batches = table.batches;
let i = 0;
const oneShot: Iterable<RecordBatch> & Iterator<RecordBatch> = {
[Symbol.iterator]() {
return this;
},
next() {
if (i >= batches.length) {
return { done: true, value: undefined };
}
return { done: false, value: batches[i++] };
},
};
await expect(
Scannable.fromIterable(table.schema, oneShot, { rescannable: true }),
).rejects.toThrow(/one-shot/);
});
test("Array.values() (IterableIterator) throws", async () => {
const table = makeTable();
const iter = table.batches.values();
await expect(
Scannable.fromIterable(table.schema, iter, { rescannable: true }),
).rejects.toThrow(/one-shot/);
});
test("raw iterator (only `.next`) throws", async () => {
const table = makeTable();
const batches = table.batches;
let i = 0;
const rawIter = {
next(): IteratorResult<RecordBatch> {
if (i >= batches.length) {
return { done: true, value: undefined };
}
return { done: false, value: batches[i++] };
},
};
await expect(
Scannable.fromIterable(
table.schema,
rawIter as unknown as Iterable<RecordBatch>,
{ rescannable: true },
),
).rejects.toThrow(/one-shot/);
});
// Edge: null/undefined must not crash the detection helper. The
// null check belongs to `normalizeIterator` and only fires when a
// scan starts.
test("null input does not crash detection at construction", async () => {
const schema = makeTable().schema;
await expect(
Scannable.fromIterable(
schema,
null as unknown as Iterable<RecordBatch>,
{
rescannable: true,
},
),
).resolves.toBeDefined();
});
test("undefined input does not crash detection at construction", async () => {
const schema = makeTable().schema;
await expect(
Scannable.fromIterable(
schema,
undefined as unknown as Iterable<RecordBatch>,
{ rescannable: true },
),
).resolves.toBeDefined();
});
// Default (rescannable omitted) skips the check entirely, so even
// pathological inputs construct without throwing here.
test("rescannable omitted skips detection entirely (generator passes)", async () => {
const table = makeTable();
function* generator(): Generator<RecordBatch> {
for (const batch of table.batches) yield batch;
}
const scannable = await Scannable.fromIterable(
table.schema,
generator(),
);
expect(scannable.rescannable).toBe(false);
});
test("rescannable: false explicit skips detection entirely (generator passes)", async () => {
const table = makeTable();
function* generator(): Generator<RecordBatch> {
for (const batch of table.batches) yield batch;
}
const scannable = await Scannable.fromIterable(
table.schema,
generator(),
{ rescannable: false },
);
expect(scannable.rescannable).toBe(false);
});
});
});
describe("fromFactory", () => {
test("defaults rescannable=true and does not invoke the factory eagerly", async () => {
const table = makeTable();
const factory = jest.fn(() => table.batches);
const scannable = await Scannable.fromFactory(table.schema, factory);
expect(scannable.schema).toBe(table.schema);
expect(scannable.rescannable).toBe(true);
expect(factory).not.toHaveBeenCalled();
});
test("honors rescannable and numRows overrides", async () => {
const table = makeTable();
const scannable = await Scannable.fromFactory(
table.schema,
() => table.batches,
{ numRows: 7, rescannable: false },
);
expect(scannable.numRows).toBe(7);
expect(scannable.rescannable).toBe(false);
});
});
describe("validation", () => {
test("throws when numRows is negative", async () => {
await expect(
Scannable.fromFactory(makeTable().schema, () => [], { numRows: -1 }),
).rejects.toThrow(/non-negative/);
});
test("throws when numRows is not an integer", async () => {
await expect(
Scannable.fromFactory(makeTable().schema, () => [], { numRows: 3.5 }),
).rejects.toThrow(/integer/);
});
});
describe("native handle", () => {
test("exposes a native handle via inner", async () => {
const scannable = await Scannable.fromTable(makeTable());
expect(scannable.inner).toBeDefined();
expect(typeof scannable.inner).toBe("object");
expect(scannable.inner).not.toBeNull();
});
});
// Schema-variety construction tests. Each asserts that construction
// succeeds against a richer Arrow schema, which transitively exercises
// schema serialization and the Rust-side `ipc_file_to_schema` for types
// beyond flat primitives.
describe("schema variety", () => {
test("accepts an empty table", async () => {
const schema = new Schema([new Field("id", new Int32(), true)]);
const table = makeEmptyTable(schema);
const scannable = await Scannable.fromTable(table);
expect(scannable.numRows).toBe(0);
expect(scannable.schema).toBe(table.schema);
});
test("accepts nested struct and list columns", async () => {
const table = makeArrowTable(
[
{ id: 1, point: { x: 0, y: 0 }, tags: ["a", "b"] },
{ id: 2, point: { x: 1, y: 2 }, tags: ["c"] },
],
{ vectorColumns: {} },
);
const scannable = await Scannable.fromTable(table);
expect(scannable.schema).toBe(table.schema);
expect(scannable.numRows).toBe(2);
});
test("accepts a FixedSizeList (vector) column", async () => {
const table = makeArrowTable(
[
{ id: 1, vec: [1, 2, 3] },
{ id: 2, vec: [4, 5, 6] },
],
{ vectorColumns: { vec: { type: new Float16() } } },
);
const scannable = await Scannable.fromTable(table);
expect(scannable.schema).toBe(table.schema);
expect(scannable.numRows).toBe(2);
});
test("accepts a table with many columns", async () => {
const row: Record<string, number> = {};
for (let i = 0; i < 50; i++) row[`c${i}`] = i;
const table = makeArrowTable([row, row], { vectorColumns: {} });
const scannable = await Scannable.fromTable(table);
expect(scannable.schema.fields.length).toBe(50);
expect(scannable.numRows).toBe(2);
});
});
});
nodejs/__test__/table.test.ts
+3478 -45
View File
File diff suppressed because it is too large Load Diff
nodejs/__test__/util.test.ts
+3
View File
@@ -14,6 +14,9 @@ test.each([
[new ArrayBuffer(8), "X'0000000000000000'"],
[Buffer.from("hello"), "X'68656c6c6f'"],
["Hello 'world'", "'Hello ''world'''"],
[NaN, "CAST('NaN' AS DOUBLE)"],
[Infinity, "CAST('Infinity' AS DOUBLE)"],
[-Infinity, "CAST('-Infinity' AS DOUBLE)"],
])("toSQL(%p) === %p", (value, expected) => {
expect(toSQL(value)).toBe(expected);
});
nodejs/__test__/vector_types.test.ts
+110
View File
@@ -0,0 +1,110 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import * as tmp from "tmp";
import { type Table, connect } from "../lancedb";
import {
Field,
FixedSizeList,
Float32,
Int64,
Schema,
makeArrowTable,
} from "../lancedb/arrow";
describe("Vector query with different typed arrays", () => {
let tmpDir: tmp.DirResult;
afterEach(() => {
tmpDir?.removeCallback();
});
async function createFloat32Table(): Promise<Table> {
tmpDir = tmp.dirSync({ unsafeCleanup: true });
const db = await connect(tmpDir.name);
const schema = new Schema([
new Field("id", new Int64(), true),
new Field(
"vec",
new FixedSizeList(2, new Field("item", new Float32())),
true,
),
]);
const data = makeArrowTable(
[
{ id: 1n, vec: [1.0, 0.0] },
{ id: 2n, vec: [0.0, 1.0] },
{ id: 3n, vec: [1.0, 1.0] },
],
{ schema },
);
return db.createTable("test_f32", data);
}
it("should search with Float32Array (baseline)", async () => {
const table = await createFloat32Table();
const results = await table
.query()
.nearestTo(new Float32Array([1.0, 0.0]))
.limit(1)
.toArray();
expect(results.length).toBe(1);
expect(Number(results[0].id)).toBe(1);
});
it("should search with number[] (backward compat)", async () => {
const table = await createFloat32Table();
const results = await table
.query()
.nearestTo([1.0, 0.0])
.limit(1)
.toArray();
expect(results.length).toBe(1);
expect(Number(results[0].id)).toBe(1);
});
it("should search with Float64Array via raw path", async () => {
const table = await createFloat32Table();
const results = await table
.query()
.nearestTo(new Float64Array([1.0, 0.0]))
.limit(1)
.toArray();
expect(results.length).toBe(1);
expect(Number(results[0].id)).toBe(1);
});
it("should add multiple query vectors with Float64Array", async () => {
const table = await createFloat32Table();
const results = await table
.query()
.nearestTo(new Float64Array([1.0, 0.0]))
.addQueryVector(new Float64Array([0.0, 1.0]))
.limit(2)
.toArray();
expect(results.length).toBeGreaterThanOrEqual(2);
});
// Float16Array is only available in Node 22+; not in TypeScript's standard lib yet
const float16ArrayCtor = (globalThis as unknown as Record<string, unknown>)
.Float16Array as (new (values: number[]) => unknown) | undefined;
const hasFloat16 = float16ArrayCtor !== undefined;
const f16it = hasFloat16 ? it : it.skip;
f16it("should search with Float16Array via raw path", async () => {
const table = await createFloat32Table();
const results = await table
.query()
.nearestTo(new float16ArrayCtor!([1.0, 0.0]) as Float32Array)
.limit(1)
.toArray();
expect(results.length).toBe(1);
expect(Number(results[0].id)).toBe(1);
});
});
nodejs/biome.json
+1
View File
@@ -48,6 +48,7 @@
"noUnreachableSuper": "error",
"noUnsafeFinally": "error",
"noUnsafeOptionalChaining": "error",
"noUnusedImports": "error",
"noUnusedLabels": "error",
"noUnusedVariables": "warn",
"useIsNan": "error",
nodejs/examples/basic.test.ts
+1 -1
View File
@@ -170,7 +170,7 @@ test("basic table examples", async () => {
// --8<-- [end:create_index]
// --8<-- [start:delete_rows]
await tbl.delete('item = "fizz"');
await tbl.delete("item = 'fizz'");
// --8<-- [end:delete_rows]
// --8<-- [start:drop_table]
nodejs/examples/filtering.test.ts
+9
View File
@@ -38,5 +38,14 @@ test("filtering examples", async () => {
// --8<-- [start:sql_search]
await tbl.query().where("id = 10").limit(10).toArray();
// --8<-- [end:sql_search]
// --8<-- [start:orderby_search]
await tbl
.query()
.where("id > 10")
.orderBy({ columnName: "id", ascending: false })
.limit(5)
.toArray();
// --8<-- [end:orderby_search]
});
});
nodejs/examples/full_text_search.test.ts
+7 -1
View File
@@ -29,8 +29,14 @@ test("full text search", async () => {
const tbl = await db.createTable("myVectors", data, { mode: "overwrite" });
await tbl.createIndex("doc", {
config: lancedb.Index.fts(),
config: lancedb.Index.fts({
stem: false,
removeStopWords: true,
customStopWords: ["banana"],
}),
});
const tokens = await tbl.tokenize("apple banana", { column: "doc" });
expect(tokens.map((token) => token.text)).toEqual(["apple"]);
// --8<-- [start:full_text_search]
const result = await tbl
nodejs/examples/package-lock.json
Generated
-4903
View File
File diff suppressed because it is too large Load Diff
nodejs/examples/package.json
+7 -5
View File
@@ -8,19 +8,21 @@
"//1": "--experimental-vm-modules is needed to run jest with sentence-transformers",
"//2": "--testEnvironment is needed to run jest with sentence-transformers",
"//3": "See: https://github.com/huggingface/transformers.js/issues/57",
"test": "node --experimental-vm-modules node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose",
"//4": "jest is invoked by its JS entry, not node_modules/.bin/jest: under pnpm that path is a shell shim, which `node` cannot execute",
"test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --testEnvironment jest-environment-node-single-context --verbose",
"lint": "biome check *.ts && biome format *.ts",
"lint-ci": "biome ci .",
"lint-fix": "biome check --write *.ts && npm run format",
"lint-fix": "biome check --write *.ts && pnpm format",
"format": "biome format --write *.ts"
},
"author": "Lance Devs",
"license": "Apache-2.0",
"packageManager": "pnpm@11.1.1",
"dependencies": {
"@huggingface/transformers": "^3.0.2",
"@huggingface/transformers": "3.0.2",
"@lancedb/lancedb": "file:../dist",
"openai": "^4.29.2",
"sharp": "^0.33.5"
"openai": "4.29.2",
"sharp": "0.33.5"
},
"devDependencies": {
"@biomejs/biome": "^1.7.3",
nodejs/examples/pnpm-lock.yaml
Generated
+3466
View File
File diff suppressed because it is too large Load Diff
nodejs/examples/pnpm-workspace.yaml
+13
View File
@@ -0,0 +1,13 @@
# Block resolution of versions less than 24h old (Shai-Hulud window).
# This is the pnpm 11 default but pinned here so it's visible to
# reviewers and survives a future pnpm major flipping the default.
minimumReleaseAge: 1440
# Fail install if a transitive dep tries to run an unapproved script.
strictDepBuilds: true
allowBuilds:
'@biomejs/biome': true
onnxruntime-node: true
protobufjs: true
sharp: true
nodejs/jest.config.js
+9
View File
@@ -1,3 +1,12 @@
const path = require("node:path");
// Set the real process environment before Jest creates its sandbox and workers.
// Assigning process.env inside a test does not reach Rust's std::env.
process.env.LANCEDB_OAUTH_BROWSER =
process.platform === "win32"
? path.join(__dirname, "__test__/fixtures/oauth_browser.cmd")
: "/usr/bin/true";
/** @type {import('ts-jest').JestConfigWithTsJest} */
module.exports = {
preset: "ts-jest",
nodejs/lancedb/arrow.ts
+551 -290
View File
File diff suppressed because it is too large Load Diff
nodejs/lancedb/arrow_type.ts
+40
View File
@@ -0,0 +1,40 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import {
type DataType,
Float32,
Float64,
Int8,
Int16,
Int32,
Uint8,
Uint16,
Uint32,
} from "apache-arrow";
/**
* Map a JS TypedArray instance to the corresponding Arrow element type and
* length. Returns undefined when the view is not a supported TypedArray.
*/
export function typedArrayToArrowType(
value: ArrayBufferView,
): { elementType: DataType; length: number } | undefined {
if (value instanceof Float32Array)
return { elementType: new Float32(), length: value.length };
if (value instanceof Float64Array)
return { elementType: new Float64(), length: value.length };
if (value instanceof Uint8Array)
return { elementType: new Uint8(), length: value.length };
if (value instanceof Uint16Array)
return { elementType: new Uint16(), length: value.length };
if (value instanceof Uint32Array)
return { elementType: new Uint32(), length: value.length };
if (value instanceof Int8Array)
return { elementType: new Int8(), length: value.length };
if (value instanceof Int16Array)
return { elementType: new Int16(), length: value.length };
if (value instanceof Int32Array)
return { elementType: new Int32(), length: value.length };
return undefined;
}
nodejs/lancedb/blob.ts
+566
View File
@@ -0,0 +1,566 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { Field, LargeBinary, Struct, Uint64, Utf8 } from "apache-arrow";
import { BlobFile as NativeBlobFile } from "./native";
const BLOB_V2_EXTENSION_NAME = "lance.blob.v2";
const INLINE_SIZE_THRESHOLD_KEY = "lance-encoding:blob-inline-size-threshold";
const DEDICATED_SIZE_THRESHOLD_KEY =
"lance-encoding:blob-dedicated-size-threshold";
const PACK_FILE_SIZE_THRESHOLD_KEY =
"lance-encoding:blob-pack-file-size-threshold";
/**
* Bytes accepted for a blob value. `ArrayBuffer` is wrapped without copying.
* `Blob` and `File` are read with `arrayBuffer()`, which only the async write
* paths ({@link Connection.createTable}, {@link Table.add},
* {@link Table.mergeInsert}) can do.
*/
export type BlobData = Buffer | Uint8Array | ArrayBuffer | Blob;
/** A URI accepted for a blob value. A `URL` is stored as its `href`. */
export type BlobUri = string | URL;
/**
* A value accepted for a `lance.blob.v2` column (see {@link blob}).
*
* Either inline bytes, a URI pointing at external bytes, or a struct that sets
* exactly one of `data` or `uri`. `null` and `undefined` write a null blob.
* External byte ranges require both
* `position` (non-negative) and `size` (positive). Use safe integer numbers or
* `bigint` values for these unsigned 64-bit byte counts.
*
* @example
* ```ts
* import { pathToFileURL } from "node:url";
* import type { BlobInput } from "@lancedb/lancedb";
*
* const rows: { id: bigint; image: BlobInput }[] = [
* { id: 1n, image: await (await fetch(url)).arrayBuffer() },
* { id: 2n, image: pathToFileURL("/data/cat.png") },
* { id: 3n, image: { data: new Blob(["hello"]) } },
* ];
* await table.add(rows);
* ```
*/
export type BlobInput =
| BlobData
| BlobUri
| { data: BlobData; uri?: null; position?: null; size?: null }
| { data?: null; uri: BlobUri; position?: null; size?: null }
| {
data?: null;
uri: BlobUri;
position: number | bigint;
size: number | bigint;
}
| null
| undefined;
/** A blob value normalized to the columns of the blob struct. */
export type BlobValue = {
data: Buffer | Uint8Array | null;
uri: string | null;
position?: bigint;
size?: bigint;
};
export type BlobOptions = {
/** Defaults to true. */
nullable?: boolean;
/**
* Max payload bytes kept inline in the data file. Zero is allowed. Must be a
* safe integer.
*/
inlineSizeThreshold?: number;
/**
* Max payload bytes stored in a packed sidecar before a dedicated file. Must
* be a positive safe integer.
*/
dedicatedSizeThreshold?: number;
/**
* Max bytes in one packed sidecar before starting another. Must be a positive
* safe integer.
*/
packFileSizeThreshold?: number;
};
/**
* Declares a `lance.blob.v2` column.
* Uses nullable `data`, `uri`, `position`, and `size` fields for inline bytes,
* whole external files, and external byte ranges. Set both `position` and
* positive `size` for a range, or omit both for a whole file.
*
* Existing two-field columns remain writable. Appends and merge inserts fill
* missing range fields with nulls for four-field targets and drop all-null
* range fields for two-field targets. Non-null ranges require a four-field
* target. Overwrite with Arrow input replaces the table's schema; row objects
* are converted using the current table schema.
*
* Query results are descriptors, not payload bytes. Use {@link Table.fetchBlobs}
* or {@link Table.fetchBlobFiles} to read bytes.
* Blobs inside lists can be written and queried as descriptors, but cannot
* currently be fetched because a row ID does not identify a list element.
*
* @example
* ```ts
* import { readFile } from "node:fs/promises";
* import { Field, Int64, Schema } from "apache-arrow";
* import { blob, connect } from "@lancedb/lancedb";
*
* const db = await connect("./data");
* const video = await readFile("clip.mp4");
* const table = await db.createTable(
* "videos",
* [{ id: 1n, video }],
* {
* schema: new Schema([
* new Field("id", new Int64()),
* blob("video"),
* ]),
* },
* );
*
* const rows = await table.query().select(["id"]).withRowId().toArray();
* const rowIds = rows.map((row) => row._rowid as bigint);
* const bytes = await table.fetchBlobs("video", rowIds);
*
* const [handle] = await table.fetchBlobFiles("video", rowIds);
* const size = handle!.size();
* const header = await handle!.readRange(0n, size < 65536n ? size : 65536n);
* ```
*/
export function blob(name: string, options: BlobOptions = {}): Field {
const metadata = new Map<string, string>([
["ARROW:extension:name", BLOB_V2_EXTENSION_NAME],
]);
setThreshold(
metadata,
INLINE_SIZE_THRESHOLD_KEY,
"inlineSizeThreshold",
options.inlineSizeThreshold,
0,
);
setThreshold(
metadata,
DEDICATED_SIZE_THRESHOLD_KEY,
"dedicatedSizeThreshold",
options.dedicatedSizeThreshold,
1,
);
setThreshold(
metadata,
PACK_FILE_SIZE_THRESHOLD_KEY,
"packFileSizeThreshold",
options.packFileSizeThreshold,
1,
);
return new Field(
name,
new Struct([
new Field("data", new LargeBinary(), true),
new Field("uri", new Utf8(), true),
new Field("position", new Uint64(), true),
new Field("size", new Uint64(), true),
]),
options.nullable ?? true,
metadata,
);
}
/**
* Checks for the `lance.blob.v2` extension marker. Does not validate the
* field's storage type.
*/
export function isBlobField(field: Field): boolean {
return field.metadata?.get("ARROW:extension:name") === BLOB_V2_EXTENSION_NAME;
}
/**
* A lazy handle to blob bytes. Create one with {@link Table.fetchBlobFiles}.
*
* @hideconstructor
*/
export class BlobFile {
private readonly inner: NativeBlobFile;
private constructor(inner: NativeBlobFile) {
if (!(inner instanceof NativeBlobFile)) {
throw new Error("BlobFile handles come from Table.fetchBlobFiles");
}
this.inner = inner;
}
/** @ignore */
static fromNative(inner: NativeBlobFile): BlobFile {
return new BlobFile(inner);
}
/** Returns the blob size in bytes. */
size(): bigint {
return this.inner.size();
}
/**
* Reads from the cursor and advances the cursor.
*
* Reads to the end when `maxBytes` is omitted, or at most `maxBytes` bytes
* otherwise. Returns an empty buffer at the end of the blob.
* {@link BlobFile.readRange} does not move the cursor.
*/
read(maxBytes?: bigint, options?: BlobReadOptions): Promise<Buffer> {
return runWithSignal(options?.signal, (signal) =>
this.inner.read(maxBytes, signal),
);
}
/**
* Reads the half-open byte range `[start, end)`.
*
* Fails when `end` is past the blob size. Does not move the cursor.
*/
readRange(
start: bigint,
end: bigint,
options?: BlobReadOptions,
): Promise<Buffer> {
return runWithSignal(options?.signal, (signal) =>
this.inner.readRange(start, end, signal),
);
}
/**
* Reads several half-open byte ranges. Returns one buffer per range, in
* the order given.
*
* Fails when any `end` is past the blob size. Does not move the cursor.
*/
readRanges(
ranges: BlobRange[],
options?: BlobReadOptions,
): Promise<Buffer[]> {
return runWithSignal(options?.signal, (signal) =>
this.inner.readRanges(ranges, signal),
);
}
/** Moves the cursor to `position`, in bytes from the start of the blob. */
seek(position: bigint): Promise<void> {
return this.inner.seek(position);
}
/** Returns the cursor position in bytes. */
tell(): Promise<bigint> {
return this.inner.tell();
}
/**
* Releases the handle. Reads after `close()` fail. Calling it again does
* nothing.
*/
close(): Promise<void> {
return this.inner.close();
}
/** Returns true after {@link BlobFile.close}. */
isClosed(): boolean {
return this.inner.isClosed();
}
}
/** Options for blob reads. */
export type BlobReadOptions = {
/**
* Cancels the read. The call rejects with `signal.reason`, and the native
* read stops, including in-flight requests to a remote table.
*/
signal?: AbortSignal;
};
/** @ignore */
export async function runWithSignal<T>(
signal: AbortSignal | undefined,
run: (signal: AbortSignal | undefined) => Promise<T>,
): Promise<T> {
if (signal === undefined) {
return run(undefined);
}
signal.throwIfAborted();
// napi-rs replaces `onabort` on the signal it receives, so pass a private
// signal and forward the caller's abort to it.
const controller = new AbortController();
const onAbort = () => controller.abort();
signal.addEventListener("abort", onAbort, { once: true });
try {
return await run(controller.signal);
} catch (err) {
if (signal.aborted) {
throw signal.reason;
}
throw err;
} finally {
signal.removeEventListener("abort", onAbort);
}
}
/** A half-open byte range `[start, end)` for {@link BlobFile.readRanges}. */
export type BlobRange = {
start: bigint;
end: bigint;
};
/**
* Rewrites the synchronous-only widenings of {@link BlobInput} (`ArrayBuffer`,
* `URL`) into `Uint8Array` and URI strings, keeping the value's shape. Other
* values are returned unchanged and validated later by {@link coerceBlobValue}.
*
* Throws on `Blob` / `File`, which must be read into bytes first.
*/
export function normalizeBlobInput(value: unknown): unknown {
if (blobToRead(value) !== undefined) {
throw new Error(
"Blob and File values must be read asynchronously. Pass them to " +
"createTable, add, or mergeInsert, or convert them with " +
"`new Uint8Array(await value.arrayBuffer())`",
);
}
if (value instanceof ArrayBuffer) {
return new Uint8Array(value);
}
if (value instanceof URL) {
return value.href;
}
if (isBlobStruct(value)) {
// Read `data` and `uri` explicitly: spreading a class instance would drop
// fields defined as getters on its prototype.
const out: Record<string, unknown> = {
...value,
data: value.data,
uri: value.uri,
position: value.position,
size: value.size,
};
let changed = false;
if (value.data instanceof ArrayBuffer) {
out.data = new Uint8Array(value.data);
changed = true;
}
if (value.uri instanceof URL) {
out.uri = value.uri.href;
changed = true;
}
return changed ? out : value;
}
return value;
}
/**
* Returns the `Blob` / `File` a blob value needs read, either the value itself
* or its `data` field, or `undefined` when there is nothing to read.
*/
export function blobToRead(value: unknown): Blob | undefined {
if (isBlobLike(value)) {
return value;
}
if (isBlobStruct(value) && isBlobLike(value.data)) {
return value.data;
}
return undefined;
}
/**
* Replaces the `Blob` that {@link blobToRead} found in `value` with `bytes`,
* keeping the value's shape.
*/
export function withBlobBytes(value: unknown, bytes: Uint8Array): unknown {
if (isBlobLike(value)) {
return bytes;
}
const record = value as Record<string, unknown>;
return {
...record,
uri: record.uri,
position: record.position,
size: record.size,
data: bytes,
};
}
export function coerceBlobValue(
input: unknown,
field?: Field,
): BlobValue | null {
const value = normalizeBlobInput(input);
if (value == null) {
return null;
}
if (isBlobBytes(value)) {
return { data: value, uri: null };
}
if (ArrayBuffer.isView(value)) {
throw new Error(
"Blob data must be Buffer, Uint8Array, ArrayBuffer, or Blob",
);
}
if (typeof value === "string") {
if (value === "") {
throw new Error("Blob uri cannot be empty");
}
return { data: null, uri: value };
}
if (isBlobStruct(value)) {
const record = value;
if (!("data" in record) && !("uri" in record)) {
throw new Error(
"Blob struct values must include a 'data' or 'uri' field",
);
}
const unknownKeys = Object.keys(record).filter(
(key) => !["data", "uri", "position", "size"].includes(key),
);
if (unknownKeys.length > 0) {
throw new Error(
`Blob struct values only support 'data', 'uri', 'position', and 'size', got ${unknownKeys
.map((key) => `'${key}'`)
.join(", ")}`,
);
}
const uri = record.uri;
if (uri === "") {
throw new Error("Blob uri cannot be empty");
}
if (uri != null && typeof uri !== "string") {
throw new Error(
`Blob uri must be a string, URL, or null, got ${typeof uri}`,
);
}
const data = record.data;
if (data != null && !isBlobBytes(data)) {
throw new Error(
"Blob data must be Buffer, Uint8Array, ArrayBuffer, Blob, or null",
);
}
const bytes = (data as Buffer | Uint8Array | null | undefined) ?? null;
const uriValue = uri ?? null;
if ((bytes == null) === (uriValue == null)) {
throw new Error(
"Blob struct values must set exactly one of 'data' or 'uri'",
);
}
const hasPosition = record.position != null;
const hasSize = record.size != null;
if (hasPosition !== hasSize) {
throw new Error(
"External blob must set both position and size, or neither",
);
}
if (hasPosition) {
if (uriValue == null) {
throw new Error("External blob byte ranges require a uri");
}
const position = blobRangeInteger(record.position, "position", 0n);
const size = blobRangeInteger(record.size, "size", 1n);
if (position + size > 0xffffffffffffffffn) {
throw new Error("External blob range overflows uint64");
}
if (
field !== undefined &&
(!(field.type instanceof Struct) ||
["position", "size"].some(
(name) =>
!field.type.children.some((child: Field) => child.name === name),
))
) {
throw new Error(
"Blob storage schema cannot store byte ranges. Use a four-field blob() schema to store position and size",
);
}
return { data: null, uri: uriValue, position, size };
}
return { data: bytes, uri: uriValue };
}
throw new Error(
"Blob column values must be Buffer, Uint8Array, ArrayBuffer, Blob, a URI string or URL, null, or { data?, uri?, position?, size? }",
);
}
function blobRangeInteger(
value: unknown,
name: string,
minimum: bigint,
): bigint {
if (
typeof value !== "bigint" &&
(typeof value !== "number" || !Number.isSafeInteger(value))
) {
throw new Error(`Blob ${name} must be a safe integer or bigint`);
}
const integer = BigInt(value);
if (integer < minimum || integer > 0xffffffffffffffffn) {
throw new Error(
`Blob ${name} must be ${minimum === 0n ? "non-negative" : "positive"} and fit in uint64`,
);
}
return integer;
}
function isBlobLike(value: unknown): value is Blob {
return typeof Blob !== "undefined" && value instanceof Blob;
}
/**
* Whether `value` is a `{ data?, uri? }` struct: any plain object, or any
* other object with a `data` or `uri` field (such as a class instance), apart
* from the binary, `Blob`, and `URL` values that are blob inputs themselves.
*/
function isBlobStruct(value: unknown): value is Record<string, unknown> {
if (
typeof value !== "object" ||
value === null ||
Array.isArray(value) ||
ArrayBuffer.isView(value) ||
value instanceof ArrayBuffer ||
(typeof SharedArrayBuffer !== "undefined" &&
value instanceof SharedArrayBuffer) ||
isBlobLike(value) ||
value instanceof URL
) {
return false;
}
const prototype = Object.getPrototypeOf(value);
return (
prototype === Object.prototype ||
prototype === null ||
"data" in value ||
"uri" in value
);
}
function isBlobBytes(value: unknown): value is Buffer | Uint8Array {
return Buffer.isBuffer(value) || value instanceof Uint8Array;
}
function setThreshold(
metadata: Map<string, string>,
key: string,
optionName: string,
value: number | undefined,
minimum: number,
): void {
if (value === undefined) {
return;
}
if (!Number.isSafeInteger(value)) {
throw new Error(`${optionName} must be a safe integer`);
}
if (value < minimum) {
throw new Error(
minimum <= 0
? `${optionName} must be non-negative`
: `${optionName} must be positive`,
);
}
metadata.set(key, String(value));
}
nodejs/lancedb/catalog.ts
+153
View File
@@ -0,0 +1,153 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { Connection, LocalConnection } from "./connection";
import { HeaderProvider } from "./header";
import {
JsHeaderProvider,
Catalog as NativeCatalog,
CatalogOptions as NativeCatalogOptions,
DatabaseNames as NativeDatabaseNames,
} from "./native.js";
import { OAuthConfig } from "./oauth";
/** Options shared by a catalog and the database connections it returns. */
export interface CatalogOptions
extends Omit<NativeCatalogOptions, "oauthConfig"> {
oauthConfig?: OAuthConfig;
/** Called for each request to supply authentication headers. */
headerProvider?:
| HeaderProvider
| (() => Record<string, string> | Promise<Record<string, string>>);
}
/**
* A lazy async iterator of database names with pagination state.
* Returned by {@link Catalog.listDatabases}.
*/
export class DatabaseNames implements AsyncIterableIterator<string> {
/** @hidden */
constructor(private readonly inner: NativeDatabaseNames) {}
[Symbol.asyncIterator](): AsyncIterableIterator<string> {
return this;
}
/** Fetch the next name, requesting another page only when needed. */
async next(): Promise<IteratorResult<string>> {
const value = await this.inner.next();
return value == null
? { done: true, value: undefined }
: { done: false, value };
}
/** Number of names available without another REST request. */
numPageResults(): number {
return this.inner.numPageResults();
}
/**
* Token for the next REST request. Initially this is the supplied starting token.
* Returns undefined after the final page is fetched, even if names remain cached.
* Drain the cache before saving a token to avoid skipping names when resuming.
* A failed request terminates iteration and retains its token for a new iterator.
*/
pageToken(): string | undefined {
return this.inner.pageToken() ?? undefined;
}
}
/** A remote catalog manages databases through the server's root namespace. */
export class Catalog {
/** @hidden */
constructor(private readonly inner: NativeCatalog) {}
/** The root namespace endpoint. */
get uri(): string {
return this.inner.uri;
}
/** Create a database, or open an existing database when existOk is true. */
async createDatabase(
name: string,
options: { existOk?: boolean } = {},
): Promise<Connection> {
return new LocalConnection(
await this.inner.createDatabase(name, options.existOk),
);
}
/** Connect to an existing database by its logical name. */
async connectDatabase(name: string): Promise<Connection> {
return new LocalConnection(await this.inner.connectDatabase(name));
}
/** Drop an empty database. The server rejects nonempty databases. */
async dropDatabase(
name: string,
options: { ignoreMissing?: boolean } = {},
): Promise<void> {
await this.inner.dropDatabase(name, options.ignoreMissing);
}
/**
* Iterate lazily over all database names using `for await...of`.
* pageToken resumes from a saved token; omitted starts at the beginning.
* pageLimit limits each REST response, not the total; omitted uses the server default.
* Request errors are raised during iteration and terminate the iterator.
*
* @example
* ```ts
* for await (const name of catalog.listDatabases({ pageLimit: 20 })) {
* console.log(name);
* }
* ```
*/
listDatabases(
options: { pageLimit?: number; pageToken?: string } = {},
): DatabaseNames {
if (
options.pageLimit !== undefined &&
(!Number.isInteger(options.pageLimit) ||
options.pageLimit <= 0 ||
options.pageLimit > 2147483647)
) {
throw new Error(
"Database list limit must be an integer between 1 and 2147483647",
);
}
return new DatabaseNames(
this.inner.listDatabases(options.pageToken, options.pageLimit),
);
}
}
/**
* Connect to an HTTP(S) catalog endpoint. Catalog requests omit database-selection
* headers; opened database connections inherit authentication and client options.
*
* @example
* ```ts
* const catalog = await connectCatalog("https://my-server.example", { apiKey: "secret" });
* const db = await catalog.createDatabase("analytics", { existOk: true });
* for await (const name of catalog.listDatabases({ pageLimit: 20 })) {
* console.log(name);
* }
* ```
*/
export async function connectCatalog(
endpoint: string,
options: CatalogOptions = {},
): Promise<Catalog> {
const { headerProvider, ...nativeOptions } = options;
const provider = headerProvider
? new JsHeaderProvider(async () =>
typeof headerProvider === "function"
? headerProvider()
: headerProvider.getHeaders(),
)
: undefined;
return new Catalog(
await NativeCatalog.new(endpoint, nativeOptions, provider),
);
}
nodejs/lancedb/connection.ts
+697 -59
View File
@@ -3,12 +3,12 @@
import {
Data,
Schema,
SchemaLike,
TableLike,
fromTableToStreamBuffer,
isArrowTable,
makeArrowTable,
resolveBlobInputs,
} from "./arrow";
import {
Table as ArrowTable,
@@ -16,7 +16,31 @@ import {
makeEmptyTable,
} from "./arrow";
import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
import { Job } from "./job";
import { Listing, ListingOptions, validateListingOptions } from "./listing";
import {
MaterializedView,
MaterializedViewSelect,
normalizeSelect,
validateNonNegativeInteger,
} from "./materialized_view";
import { Connection as LanceDbConnection } from "./native";
import type {
CreateNamespaceResponse,
DescribeNamespaceResponse,
DropNamespaceResponse,
JobInfo,
ListNamespacesResponse,
ListTablesResponse,
} from "./native";
import { ViewDescription, viewDescriptionFromNative } from "./view";
export type {
CreateNamespaceResponse,
DescribeNamespaceResponse,
DropNamespaceResponse,
ListNamespacesResponse,
ListTablesResponse,
};
import { sanitizeTable } from "./sanitize";
import { LocalTable, Table } from "./table";
@@ -43,7 +67,7 @@ export interface CreateTableOptions {
* Options already set on the connection will be inherited by the table,
* but can be overridden here.
*
* The available options are described at https://lancedb.github.io/lancedb/guides/storage/
* The available options are described at https://docs.lancedb.com/storage/
*/
storageOptions?: Record<string, string>;
@@ -73,13 +97,27 @@ export interface CreateTableOptions {
}
export interface OpenTableOptions {
/**
* Open the table scoped to this branch instead of the default branch.
*
* Reads and writes on the returned table operate in the branch's context.
*/
branch?: string;
/**
* Open the table pinned to this version, producing a read-only view.
*
* Composes with {@link OpenTableOptions.branch}: when both are set, opens
* that branch at the version; otherwise opens `main` at the version. Call
* `checkoutLatest` to return to a writable state.
*/
version?: number;
/**
* Configuration for object storage.
*
* Options already set on the connection will be inherited by the table,
* but can be overridden here.
*
* The available options are described at https://lancedb.github.io/lancedb/guides/storage/
* The available options are described at https://docs.lancedb.com/storage/
*/
storageOptions?: Record<string, string>;
/**
@@ -99,6 +137,10 @@ export interface OpenTableOptions {
indexCacheSize?: number;
}
/**
* @deprecated Use {@link ListTablesOptions} with {@link Connection.listTables}
* instead.
*/
export interface TableNamesOptions {
/**
* If present, only return names that come lexicographically after the
@@ -108,9 +150,63 @@ export interface TableNamesOptions {
* the last table name from the previous page.
*/
startAfter?: string;
/** The maximum number of names to return. Omitted returns all names; zero returns none. */
limit?: number;
}
export interface ListTablesOptions {
/**
* Token from a previous response, to resume listing where it left off.
*
* The token is opaque: it carries whatever the database needs to resume, and
* callers should not construct or interpret one.
*/
pageToken?: string;
/**
* An upper bound on how many tables to return.
* Defaults to 100. Zero returns an empty page without a continuation token.
*
* A page may hold fewer than this and still not be the last one, so keep
* going while the response carries a page token rather than while pages are
* full.
*/
limit?: number;
}
export interface ListNamespacesOptions {
/** Token from a previous response for pagination. */
pageToken?: string;
/** An optional limit to the number of results to return. */
limit?: number;
}
export interface CreateNamespaceOptions {
/** Creation mode. */
mode?: "create" | "exist_ok" | "overwrite";
/** Properties to set on the new namespace. */
properties?: Record<string, string>;
}
export interface DropNamespaceOptions {
/** Whether to skip if the namespace doesn't exist, or fail. */
mode?: "skip" | "fail";
/** Refuse to drop if non-empty (restrict) or drop recursively (cascade). */
behavior?: "restrict" | "cascade";
}
export interface RenameTableOptions {
/**
* The namespace path of the table being renamed. Defaults to the root
* namespace (`[]`) when omitted.
*/
namespacePath?: string[];
/**
* The namespace path to move the table to as part of the rename. When
* omitted the table stays in `namespacePath`.
*/
newNamespacePath?: string[];
}
/**
* A LanceDB Connection that allows you to open tables and create new ones.
*
@@ -161,31 +257,185 @@ export abstract class Connection {
* @param {Partial<TableNamesOptions>} options - options to control the
* paging / start point (backwards compatibility)
*
* @deprecated Use {@link Connection.listTables} instead.
*/
abstract tableNames(options?: Partial<TableNamesOptions>): Promise<string[]>;
/**
* List all the table names in this database.
*
* Tables will be returned in lexicographical order.
* @param {string[]} namespace - The namespace to list tables from (defaults to root namespace)
* @param {string[]} namespacePath - The namespace path to list tables from (defaults to root namespace)
* @param {Partial<TableNamesOptions>} options - options to control the
* paging / start point
*
* @deprecated Use {@link Connection.listTables} instead.
*/
abstract tableNames(
namespace?: string[],
namespacePath?: string[],
options?: Partial<TableNamesOptions>,
): Promise<string[]>;
/**
* List a page of the tables in this database.
*
* To retrieve the tables after the page, pass the `pageToken` the response
* carries back in. A page can be shorter than `limit` without being the last
* one, so walk until a response carries no page token:
*
* ```ts
* const names = [];
* let pageToken = undefined;
* do {
* const page = await conn.listTables({ pageToken, limit: 100 });
* names.push(...page.tables);
* pageToken = page.pageToken;
* } while (pageToken);
* ```
*
* @param {Partial<ListTablesOptions>} options - Pagination options
* (`pageToken`, `limit`).
* @returns {Promise<ListTablesResponse>} A page of table names and an
* optional token for the tables after it.
*/
abstract listTables(
options?: Partial<ListTablesOptions>,
): Promise<ListTablesResponse>;
/**
* List a page of the tables in this database.
*
* @param {string[]} namespacePath - The namespace path to list tables from
* (defaults to root namespace)
* @param {Partial<ListTablesOptions>} options - Pagination options
* (`pageToken`, `limit`).
* @returns {Promise<ListTablesResponse>} A page of table names and an
* optional token for the tables after it.
*/
abstract listTables(
namespacePath?: string[],
options?: Partial<ListTablesOptions>,
): Promise<ListTablesResponse>;
/**
* Open a table in the database.
* @param {string} name - The name of the table
* @param {string[]} namespace - The namespace of the table (defaults to root namespace)
* @param {string[]} namespacePath - The namespace path of the table (defaults to root namespace)
* @param {Partial<OpenTableOptions>} options - Additional options
*/
/**
* Define a materialized view named `name` over the table `source`.
*
* The view is populated before creation returns. Set `withNoData` to create
* only its definition and empty backing table. The view is a normal table:
* it can be queried, indexed and searched, and it appears in `tableNames`.
*/
abstract createMaterializedView(
name: string,
source: string,
options?: {
select?: MaterializedViewSelect;
where?: string;
limit?: number;
withNoData?: boolean;
},
): Promise<MaterializedView>;
/**
* Open the materialized view named `name`.
*
* Rejects a table that exists but is not a materialized view.
*/
abstract openMaterializedView(name: string): Promise<MaterializedView>;
/**
* The names of the materialized views in this database.
*
* Iterate lazily with `for await...of`. Local listings inspect table schemas
* one page at a time. Options control page size and the starting token.
* See {@link Listing} for pagination state and error behavior.
*/
abstract listMaterializedViews(options?: ListingOptions): Listing<string>;
/**
* Drop the materialized view named `name`.
*
* The view may become unavailable before physical cleanup finishes. Use
* {@link dropMaterializedViewAsync} to retain and wait for the cleanup job.
*
* Rejects a table that exists but is not a materialized view.
*/
abstract dropMaterializedView(
name: string,
namespacePath?: string[],
): Promise<void>;
/**
* Start dropping the materialized view named `name` and return its cleanup
* job without waiting for completion.
*
* Rejects a table that exists but is not a materialized view.
*/
abstract dropMaterializedViewAsync(
name: string,
namespacePath?: string[],
): Promise<Job>;
/**
* Create a view: a named query the database plans on every read.
*
* The query is planned once, at creation, so one that cannot be planned is
* rejected now rather than at the first read. A view holds no rows, and its
* readers see its sources as they are at read time.
*
* There is no replace: a name already taken is an error, and changing a
* view is a drop followed by a create.
*/
abstract createView(
name: string,
query: string,
namespacePath?: string[],
): Promise<ViewDescription>;
/**
* What this database records about the view named `name`: its defining
* query and the schema that query resolved to.
*/
abstract describeView(
name: string,
namespacePath?: string[],
): Promise<ViewDescription>;
/**
* Drop the view named `name` and wait for its definition to be deleted.
*
* The tables it reads are untouched: a view holds no rows of its own. Use
* {@link dropViewAsync} to retain the cleanup job instead of waiting on it.
*/
abstract dropView(name: string, namespacePath?: string[]): Promise<void>;
/**
* Start dropping the view named `name` and return the job deleting its
* definition, without waiting for completion.
*
* The name is free before this resolves. When nothing was bound to it, the
* returned job is already finished and has no id.
*/
abstract dropViewAsync(name: string, namespacePath?: string[]): Promise<Job>;
/**
* The names of the views in one namespace.
*
* Names only; a definition comes from {@link describeView}.
* Iterate lazily with `for await...of`. See {@link Listing} for pagination
* state and error behavior; options control page size and the starting token.
*/
abstract listViews(
namespacePath?: string[],
options?: ListingOptions,
): Listing<string>;
abstract openTable(
name: string,
namespace?: string[],
namespacePath?: string[],
options?: Partial<OpenTableOptions>,
): Promise<Table>;
@@ -194,7 +444,7 @@ export abstract class Connection {
* @param {object} options - The options object.
* @param {string} options.name - The name of the table.
* @param {Data} options.data - Non-empty Array of Records to be inserted into the table
* @param {string[]} namespace - The namespace to create the table in (defaults to root namespace)
* @param {string[]} namespacePath - The namespace path to create the table in (defaults to root namespace)
*
*/
abstract createTable(
@@ -202,7 +452,7 @@ export abstract class Connection {
name: string;
data: Data;
} & Partial<CreateTableOptions>,
namespace?: string[],
namespacePath?: string[],
): Promise<Table>;
/**
* Creates a new Table and initialize it with new data.
@@ -221,13 +471,13 @@ export abstract class Connection {
* @param {string} name - The name of the table.
* @param {Record<string, unknown>[] | TableLike} data - Non-empty Array of Records
* to be inserted into the table
* @param {string[]} namespace - The namespace to create the table in (defaults to root namespace)
* @param {string[]} namespacePath - The namespace path to create the table in (defaults to root namespace)
* @param {Partial<CreateTableOptions>} options - Additional options
*/
abstract createTable(
name: string,
data: Record<string, unknown>[] | TableLike,
namespace?: string[],
namespacePath?: string[],
options?: Partial<CreateTableOptions>,
): Promise<Table>;
@@ -246,28 +496,188 @@ export abstract class Connection {
* Creates a new empty Table
* @param {string} name - The name of the table.
* @param {Schema} schema - The schema of the table
* @param {string[]} namespace - The namespace to create the table in (defaults to root namespace)
* @param {string[]} namespacePath - The namespace path to create the table in (defaults to root namespace)
* @param {Partial<CreateTableOptions>} options - Additional options
*/
abstract createEmptyTable(
name: string,
schema: import("./arrow").SchemaLike,
namespace?: string[],
namespacePath?: string[],
options?: Partial<CreateTableOptions>,
): Promise<Table>;
/**
* Drop an existing table.
* @param {string} name The name of the table to drop.
* @param {string[]} namespace The namespace of the table (defaults to root namespace).
* @param {string[]} namespacePath The namespace path of the table (defaults to root namespace).
*/
abstract dropTable(name: string, namespace?: string[]): Promise<void>;
abstract dropTable(name: string, namespacePath?: string[]): Promise<void>;
/**
* Start dropping a table and return its cleanup job.
*
* The table may become unavailable before its data files are removed. Wait
* on the returned job to know when cleanup has finished.
*/
abstract dropTableAsync(name: string, namespacePath?: string[]): Promise<Job>;
/**
* Drop all tables in the database.
* @param {string[]} namespace The namespace to drop tables from (defaults to root namespace).
* @param {string[]} namespacePath The namespace path to drop tables from (defaults to root namespace).
*/
abstract dropAllTables(namespace?: string[]): Promise<void>;
abstract dropAllTables(namespacePath?: string[]): Promise<void>;
/**
* Describe a namespace, returning its properties.
*
* @param {string[]} namespacePath - The namespace path to describe, in
* parent → child order, e.g. `["analytics", "sales"]`.
* @returns {Promise<DescribeNamespaceResponse>} The namespace's properties
* (may be undefined if the namespace has none).
*/
abstract describeNamespace(
namespacePath: string[],
): Promise<DescribeNamespaceResponse>;
/**
* List the immediate child namespaces under the given parent.
*
* Results may be paginated. To retrieve subsequent pages, pass the
* `pageToken` returned by a previous call.
*
* @param {string[]} namespacePath - The parent namespace path. Defaults
* to the root namespace if omitted.
* @param {Partial<ListNamespacesOptions>} options - Pagination options
* (`pageToken`, `limit`).
* @returns {Promise<ListNamespacesResponse>} Child namespace names and
* an optional token for fetching the next page.
*/
abstract listNamespaces(
namespacePath?: string[],
options?: Partial<ListNamespacesOptions>,
): Promise<ListNamespacesResponse>;
/**
* Create a new namespace at the given path.
*
* @param {string[]} namespacePath - The namespace path to create.
* @param {Partial<CreateNamespaceOptions>} options - Creation `mode`
* ("create" | "exist_ok" | "overwrite") and optional `properties`
* to attach to the namespace.
* @returns {Promise<CreateNamespaceResponse>} The properties of the
* created namespace and an optional transaction id.
*/
abstract createNamespace(
namespacePath: string[],
options?: Partial<CreateNamespaceOptions>,
): Promise<CreateNamespaceResponse>;
/**
* Drop a namespace.
*
* Use `behavior: "cascade"` to also drop everything contained in the
* namespace (sub-namespaces and tables). The default `"restrict"`
* behavior refuses to drop a non-empty namespace.
*
* @param {string[]} namespacePath - The namespace path to drop.
* @param {Partial<DropNamespaceOptions>} options - `mode` ("skip" | "fail"
* for missing-namespace handling) and `behavior` ("restrict" | "cascade").
* @returns {Promise<DropNamespaceResponse>} Any properties returned by
* the server and an optional transaction id.
*/
abstract dropNamespace(
namespacePath: string[],
options?: Partial<DropNamespaceOptions>,
): Promise<DropNamespaceResponse>;
/**
* Clone a table from a source table.
*
* A shallow clone creates a new table that shares the underlying data files
* with the source table but has its own independent manifest. This allows
* both the source and cloned tables to evolve independently while initially
* sharing the same data, deletion, and index files.
*
* @param {string} targetTableName - The name of the target table to create.
* @param {string} sourceUri - The URI of the source table to clone from.
* @param {object} options - Clone options.
* @param {string[]} options.targetNamespacePath - The namespace path for the target table (defaults to root namespace).
* @param {number} options.sourceVersion - The version of the source table to clone.
* @param {string} options.sourceTag - The tag of the source table to clone.
* @param {boolean} options.isShallow - Whether to perform a shallow clone (defaults to true).
*/
abstract cloneTable(
targetTableName: string,
sourceUri: string,
options?: {
targetNamespacePath?: string[];
sourceVersion?: number;
sourceTag?: string;
isShallow?: boolean;
},
): Promise<Table>;
/**
* Rename a table.
*
* Currently only supported by LanceDB Cloud. Local OSS connections and
* namespace-backed connections (via {@link connectNamespace}) reject with
* a "not supported" error.
*
* @param {string} currentName - The current name of the table.
* @param {string} newName - The new name for the table.
* @param {RenameTableOptions} options - Optional namespace paths. When
* `newNamespacePath` is omitted the table stays in `namespacePath`.
*/
abstract renameTable(
currentName: string,
newName: string,
options?: RenameTableOptions,
): Promise<void>;
/**
* Open a server-side job by id, returning a handle with its record already
* populated. Rejects when the server has no such job, the way
* {@link Connection.openTable} does for a missing table.
*
* The returned {@link Job} answers for its own state, specification,
* result, failure and event history, so there is no separate
* connection-level call for any of them.
*/
abstract openJob(jobId: string): Promise<Job>;
/**
* Iterate lazily over server-side jobs across the database's tables.
* Use `for await...of`. Options control page size and the starting token.
* See {@link Listing} for pagination state and error behavior.
*/
abstract listJobs(options?: ListingOptions): Listing<JobInfo>;
/**
* Request cancellation of a server-side job by id.
*
* Resolves to true if the server accepted the cancellation, false if no
* such job exists. Cancelling an already-terminal job is a no-op success.
*/
abstract cancelJob(jobId: string): Promise<boolean>;
/**
* Pause a server-side job by id.
*
* The job's workers drain and it stays parked until resumed. Resolves to
* "pausing", "already_paused", or "committing" -- a job finalizing its
* results cannot be parked; retry shortly.
*/
abstract pauseJob(jobId: string): Promise<string>;
/**
* Resume a paused server-side job by id.
*
* Its workers pick their work back up from checkpoints. Resolves to
* "resumed", "still_pausing" -- the pause's worker drain is not confirmed
* yet; retry shortly -- or "not_paused".
*/
abstract resumeJob(jobId: string): Promise<string>;
}
/** @hideconstructor */
@@ -293,42 +703,187 @@ export class LocalConnection extends Connection {
}
async tableNames(
namespaceOrOptions?: string[] | Partial<TableNamesOptions>,
namespacePathOrOptions?: string[] | Partial<TableNamesOptions>,
options?: Partial<TableNamesOptions>,
): Promise<string[]> {
// Detect if first argument is namespace array or options object
let namespace: string[] | undefined;
// Detect if first argument is namespacePath array or options object
let namespacePath: string[] | undefined;
let tableNamesOptions: Partial<TableNamesOptions> | undefined;
if (Array.isArray(namespaceOrOptions)) {
// First argument is namespace array
namespace = namespaceOrOptions;
if (Array.isArray(namespacePathOrOptions)) {
// First argument is namespacePath array
namespacePath = namespacePathOrOptions;
tableNamesOptions = options;
} else {
// First argument is options object (backwards compatibility)
namespace = undefined;
tableNamesOptions = namespaceOrOptions;
namespacePath = undefined;
tableNamesOptions = namespacePathOrOptions;
}
return this.inner.tableNames(
namespace ?? [],
namespacePath ?? [],
tableNamesOptions?.startAfter,
tableNamesOptions?.limit,
);
}
async createMaterializedView(
name: string,
source: string,
options?: {
select?: MaterializedViewSelect;
where?: string;
limit?: number;
withNoData?: boolean;
},
): Promise<MaterializedView> {
validateNonNegativeInteger(options?.limit, "limit");
const innerTable = await this.inner.createMaterializedView(
name,
source,
normalizeSelect(options?.select),
options?.where,
options?.limit,
options?.withNoData ?? false,
);
return new MaterializedView(new LocalTable(innerTable));
}
async openMaterializedView(name: string): Promise<MaterializedView> {
const innerTable = await this.inner.openMaterializedView(name);
return new MaterializedView(new LocalTable(innerTable));
}
listMaterializedViews(options: ListingOptions = {}): Listing<string> {
validateListingOptions(options);
return new Listing(
this.inner.listMaterializedViews(options.pageToken, options.pageLimit),
);
}
async dropMaterializedView(
name: string,
namespacePath?: string[],
): Promise<void> {
return this.inner.dropMaterializedView(name, namespacePath ?? []);
}
async dropMaterializedViewAsync(
name: string,
namespacePath?: string[],
): Promise<Job> {
return new Job(
await this.inner.dropMaterializedViewAsync(name, namespacePath ?? []),
);
}
async createView(
name: string,
query: string,
namespacePath?: string[],
): Promise<ViewDescription> {
return viewDescriptionFromNative(
await this.inner.createView(name, query, namespacePath ?? []),
);
}
async describeView(
name: string,
namespacePath?: string[],
): Promise<ViewDescription> {
return viewDescriptionFromNative(
await this.inner.describeView(name, namespacePath ?? []),
);
}
async dropView(name: string, namespacePath?: string[]): Promise<void> {
return this.inner.dropView(name, namespacePath ?? []);
}
async dropViewAsync(name: string, namespacePath?: string[]): Promise<Job> {
return new Job(await this.inner.dropViewAsync(name, namespacePath ?? []));
}
listViews(
namespacePath?: string[],
options: ListingOptions = {},
): Listing<string> {
validateListingOptions(options);
return new Listing(
this.inner.listViews(
namespacePath ?? [],
options.pageToken,
options.pageLimit,
),
);
}
async listTables(
namespacePathOrOptions?: string[] | Partial<ListTablesOptions>,
options?: Partial<ListTablesOptions>,
): Promise<ListTablesResponse> {
// Detect if first argument is namespacePath array or options object
const namespacePath = Array.isArray(namespacePathOrOptions)
? namespacePathOrOptions
: undefined;
const listTablesOptions = Array.isArray(namespacePathOrOptions)
? options
: namespacePathOrOptions;
return this.inner.listTables(
namespacePath ?? [],
listTablesOptions?.pageToken,
listTablesOptions?.limit,
);
}
async openTable(
name: string,
namespace?: string[],
namespacePath?: string[],
options?: Partial<OpenTableOptions>,
): Promise<Table> {
const innerTable = await this.inner.openTable(
name,
namespace ?? [],
namespacePath ?? [],
cleanseStorageOptions(options?.storageOptions),
options?.indexCacheSize,
);
let table: Table = new LocalTable(innerTable);
// "main" is the default branch, so treat it as no branch. On a real branch,
// scope and pin in one step (yielding "version V of branch B"); otherwise
// pin the version, if any, against main.
const branch =
options?.branch != null && options.branch !== "main"
? options.branch
: undefined;
if (branch != null) {
table = await (await table.branches()).checkout(branch, options?.version);
} else if (options?.version != null) {
await table.checkout(options.version);
}
return table;
}
async cloneTable(
targetTableName: string,
sourceUri: string,
options?: {
targetNamespacePath?: string[];
sourceVersion?: number;
sourceTag?: string;
isShallow?: boolean;
},
): Promise<Table> {
const innerTable = await this.inner.cloneTable(
targetTableName,
sourceUri,
options?.targetNamespacePath ?? [],
options?.sourceVersion ?? null,
options?.sourceTag ?? null,
options?.isShallow ?? true,
);
return new LocalTable(innerTable);
}
@@ -358,42 +913,42 @@ export class LocalConnection extends Connection {
nameOrOptions:
| string
| ({ name: string; data: Data } & Partial<CreateTableOptions>),
dataOrNamespace?: Record<string, unknown>[] | TableLike | string[],
namespaceOrOptions?: string[] | Partial<CreateTableOptions>,
dataOrNamespacePath?: Record<string, unknown>[] | TableLike | string[],
namespacePathOrOptions?: string[] | Partial<CreateTableOptions>,
options?: Partial<CreateTableOptions>,
): Promise<Table> {
if (typeof nameOrOptions !== "string" && "name" in nameOrOptions) {
// First overload: createTable(options, namespace?)
// First overload: createTable(options, namespacePath?)
const { name, data, ...createOptions } = nameOrOptions;
const namespace = dataOrNamespace as string[] | undefined;
return this._createTableImpl(name, data, namespace, createOptions);
const namespacePath = dataOrNamespacePath as string[] | undefined;
return this._createTableImpl(name, data, namespacePath, createOptions);
}
// Second overload: createTable(name, data, namespace?, options?)
// Second overload: createTable(name, data, namespacePath?, options?)
const name = nameOrOptions;
const data = dataOrNamespace as Record<string, unknown>[] | TableLike;
const data = dataOrNamespacePath as Record<string, unknown>[] | TableLike;
// Detect if third argument is namespace array or options object
let namespace: string[] | undefined;
// Detect if third argument is namespacePath array or options object
let namespacePath: string[] | undefined;
let createOptions: Partial<CreateTableOptions> | undefined;
if (Array.isArray(namespaceOrOptions)) {
// Third argument is namespace array
namespace = namespaceOrOptions;
if (Array.isArray(namespacePathOrOptions)) {
// Third argument is namespacePath array
namespacePath = namespacePathOrOptions;
createOptions = options;
} else {
// Third argument is options object (backwards compatibility)
namespace = undefined;
createOptions = namespaceOrOptions;
namespacePath = undefined;
createOptions = namespacePathOrOptions;
}
return this._createTableImpl(name, data, namespace, createOptions);
return this._createTableImpl(name, data, namespacePath, createOptions);
}
private async _createTableImpl(
name: string,
data: Data,
namespace?: string[],
namespacePath?: string[],
options?: Partial<CreateTableOptions>,
): Promise<Table> {
if (data === undefined) {
@@ -407,7 +962,7 @@ export class LocalConnection extends Connection {
name,
buf,
mode,
namespace ?? [],
namespacePath ?? [],
storageOptions,
);
@@ -417,21 +972,21 @@ export class LocalConnection extends Connection {
async createEmptyTable(
name: string,
schema: import("./arrow").SchemaLike,
namespaceOrOptions?: string[] | Partial<CreateTableOptions>,
namespacePathOrOptions?: string[] | Partial<CreateTableOptions>,
options?: Partial<CreateTableOptions>,
): Promise<Table> {
// Detect if third argument is namespace array or options object
let namespace: string[] | undefined;
// Detect if third argument is namespacePath array or options object
let namespacePath: string[] | undefined;
let createOptions: Partial<CreateTableOptions> | undefined;
if (Array.isArray(namespaceOrOptions)) {
// Third argument is namespace array
namespace = namespaceOrOptions;
if (Array.isArray(namespacePathOrOptions)) {
// Third argument is namespacePath array
namespacePath = namespacePathOrOptions;
createOptions = options;
} else {
// Third argument is options object (backwards compatibility)
namespace = undefined;
createOptions = namespaceOrOptions;
namespacePath = undefined;
createOptions = namespacePathOrOptions;
}
let mode: string = createOptions?.mode ?? "create";
@@ -454,18 +1009,97 @@ export class LocalConnection extends Connection {
name,
buf,
mode,
namespace ?? [],
namespacePath ?? [],
storageOptions,
);
return new LocalTable(innerTable);
}
async dropTable(name: string, namespace?: string[]): Promise<void> {
return this.inner.dropTable(name, namespace ?? []);
async dropTable(name: string, namespacePath?: string[]): Promise<void> {
return this.inner.dropTable(name, namespacePath ?? []);
}
async dropAllTables(namespace?: string[]): Promise<void> {
return this.inner.dropAllTables(namespace ?? []);
async dropTableAsync(name: string, namespacePath?: string[]): Promise<Job> {
return new Job(await this.inner.dropTableAsync(name, namespacePath ?? []));
}
async dropAllTables(namespacePath?: string[]): Promise<void> {
return this.inner.dropAllTables(namespacePath ?? []);
}
describeNamespace(
namespacePath: string[],
): Promise<DescribeNamespaceResponse> {
return this.inner.describeNamespace(namespacePath);
}
listNamespaces(
namespacePath?: string[],
options?: Partial<ListNamespacesOptions>,
): Promise<ListNamespacesResponse> {
return this.inner.listNamespaces(
namespacePath ?? [],
options?.pageToken,
options?.limit,
);
}
createNamespace(
namespacePath: string[],
options?: Partial<CreateNamespaceOptions>,
): Promise<CreateNamespaceResponse> {
return this.inner.createNamespace(
namespacePath,
options?.mode,
options?.properties,
);
}
dropNamespace(
namespacePath: string[],
options?: Partial<DropNamespaceOptions>,
): Promise<DropNamespaceResponse> {
return this.inner.dropNamespace(
namespacePath,
options?.mode,
options?.behavior,
);
}
async renameTable(
currentName: string,
newName: string,
options?: RenameTableOptions,
): Promise<void> {
return this.inner.renameTable(
currentName,
newName,
options?.namespacePath ?? [],
options?.newNamespacePath,
);
}
async openJob(jobId: string): Promise<Job> {
return new Job(await this.inner.openJob(jobId));
}
listJobs(options: ListingOptions = {}): Listing<JobInfo> {
validateListingOptions(options);
return new Listing(
this.inner.listJobs(options.pageToken, options.pageLimit),
);
}
async cancelJob(jobId: string): Promise<boolean> {
return this.inner.cancelJob(jobId);
}
async pauseJob(jobId: string): Promise<string> {
return this.inner.pauseJob(jobId);
}
async resumeJob(jobId: string): Promise<string> {
return this.inner.resumeJob(jobId);
}
}
@@ -525,7 +1159,11 @@ async function parseTableData(
if (isArrowTable(data)) {
table = sanitizeTable(data);
} else {
table = makeArrowTable(data as Record<string, unknown>[], options);
const records = await resolveBlobInputs(
data as Record<string, unknown>[],
options?.schema,
);
table = makeArrowTable(records, options);
}
if (streaming) {
const buf = await fromTableToStreamBuffer(
nodejs/lancedb/embedding/index.ts
+42 -2
View File
@@ -4,7 +4,15 @@
import { Field, Schema } from "../arrow";
import { sanitizeType } from "../sanitize";
import { EmbeddingFunction } from "./embedding_function";
import { EmbeddingFunctionConfig, getRegistry } from "./registry";
import {
EmbeddingFunctionConfig,
EmbeddingFunctionRegistry,
getRegistry as getGlobalRegistry,
registerBuiltIn,
} from "./registry";
type OpenAIModule = typeof import("./openai");
type TransformersModule = typeof import("./transformers");
export {
FieldOptions,
@@ -14,7 +22,39 @@ export {
EmbeddingFunctionConstructor,
} from "./embedding_function";
export * from "./registry";
export {
EmbeddingFunctionRegistry,
parseEmbeddingMetadata,
register,
} from "./registry";
export type {
CreateReturnType,
EmbeddingFunctionConfig,
EmbeddingFunctionCreate,
EmbeddingMetadataEntry,
ResolvedEmbeddingFunctionConfig,
} from "./registry";
function initializeBuiltInProviders() {
const { OpenAIEmbeddingFunction } = require("./openai") as OpenAIModule;
const { TransformersEmbeddingFunction } =
require("./transformers") as TransformersModule;
registerBuiltIn("openai", OpenAIEmbeddingFunction);
registerBuiltIn("huggingface", TransformersEmbeddingFunction);
}
/**
* Get the global embedding function registry.
*
* LanceDB built-in providers are initialized when this public API is first
* used, so importing the root package does not change automatic search
* selection for tables without embedding metadata.
*/
export function getRegistry(): EmbeddingFunctionRegistry {
initializeBuiltInProviders();
return getGlobalRegistry();
}
/**
* Create a schema with embedding functions.
nodejs/lancedb/embedding/openai.ts
+3 -2
View File
@@ -5,14 +5,13 @@ import type OpenAI from "openai";
import type { EmbeddingCreateParams } from "openai/resources/index";
import { Float, Float32 } from "../arrow";
import { EmbeddingFunction } from "./embedding_function";
import { register } from "./registry";
import { registerBuiltIn } from "./registry";
export type OpenAIOptions = {
apiKey: string;
model: EmbeddingCreateParams["model"];
};
@register("openai")
export class OpenAIEmbeddingFunction extends EmbeddingFunction<
string,
Partial<OpenAIOptions>
@@ -100,3 +99,5 @@ export class OpenAIEmbeddingFunction extends EmbeddingFunction<
return response.data[0].embedding;
}
}
registerBuiltIn("openai", OpenAIEmbeddingFunction);
nodejs/lancedb/embedding/registry.ts
+128 -33
View File
@@ -7,6 +7,10 @@ import {
} from "./embedding_function";
import "reflect-metadata";
const builtInFunctionsKey = Symbol.for(
"@lancedb/lancedb::embedding-built-in-functions::v1",
);
export type CreateReturnType<T> = T extends { init: () => Promise<void> }
? Promise<T>
: T;
@@ -59,6 +63,15 @@ export class EmbeddingFunctionRegistry {
};
}
/** @ignore */
setBuiltIn<
T extends EmbeddingFunctionConstructor = EmbeddingFunctionConstructor,
>(name: string, ctor: T): T {
this.#functions.set(name, ctor);
Reflect.defineMetadata("lancedb::embedding::name", name, ctor);
return ctor;
}
get<T extends EmbeddingFunction<unknown>>(
name: string,
): EmbeddingFunctionCreate<T> | undefined;
@@ -96,6 +109,7 @@ export class EmbeddingFunctionRegistry {
*/
reset(this: EmbeddingFunctionRegistry) {
this.#functions.clear();
getBuiltInFunctions(this).clear();
}
/**
@@ -104,41 +118,29 @@ export class EmbeddingFunctionRegistry {
async parseFunctions(
this: EmbeddingFunctionRegistry,
metadata: Map<string, string>,
): Promise<Map<string, EmbeddingFunctionConfig>> {
): Promise<Map<string, ResolvedEmbeddingFunctionConfig>> {
if (!metadata.has("embedding_functions")) {
return new Map();
} else {
type FunctionConfig = {
name: string;
sourceColumn: string;
vectorColumn: string;
model: EmbeddingFunction["TOptions"];
};
const functions = <FunctionConfig[]>(
JSON.parse(metadata.get("embedding_functions")!)
);
const items: [string, EmbeddingFunctionConfig][] = await Promise.all(
functions.map(async (f) => {
const fn = this.get(f.name);
if (!fn) {
throw new Error(`Function "${f.name}" not found in registry`);
}
const func = await this.get(f.name)!.create(f.model);
return [
f.name,
{
sourceColumn: f.sourceColumn,
vectorColumn: f.vectorColumn,
function: func,
},
];
}),
);
return new Map(items);
}
const entries = parseEmbeddingMetadata(
metadata.get("embedding_functions")!,
);
const items = await Promise.all(
entries.map(async (f): Promise<ResolvedEmbeddingFunctionConfig> => {
const fn = this.get(f.name);
if (!fn) {
throw new Error(`Function "${f.name}" not found in registry`);
}
const func = await fn.create(f.model);
return {
sourceColumn: f.sourceColumn,
vectorColumn: f.vectorColumn,
function: func,
};
}),
);
// Keyed by output column: one function may serve several columns.
return new Map(items.map((config) => [config.vectorColumn, config]));
}
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
functionToMetadata(conf: EmbeddingFunctionConfig): Record<string, any> {
@@ -195,12 +197,56 @@ export class EmbeddingFunctionRegistry {
}
}
const _REGISTRY = new EmbeddingFunctionRegistry();
function getBuiltInFunctions(registry: EmbeddingFunctionRegistry): Set<string> {
const registryWithBuiltIns = registry as EmbeddingFunctionRegistry & {
[key: symbol]: Set<string> | undefined;
};
let builtInFunctions = registryWithBuiltIns[builtInFunctionsKey];
if (builtInFunctions === undefined) {
builtInFunctions = new Set<string>();
registryWithBuiltIns[builtInFunctionsKey] = builtInFunctions;
}
return builtInFunctions;
}
// Server bundlers can load the side-effect embedding entry points and the public
// embedding API from separate module graphs. Keep their registry shared.
const registryKey = Symbol.for(
"@lancedb/lancedb::embedding-function-registry::v1",
);
const registryGlobal = globalThis as typeof globalThis & {
[key: symbol]: EmbeddingFunctionRegistry | undefined;
};
function getGlobalRegistry(): EmbeddingFunctionRegistry {
const existingRegistry = registryGlobal[registryKey];
if (existingRegistry !== undefined) {
return existingRegistry;
}
const registry = new EmbeddingFunctionRegistry();
registryGlobal[registryKey] = registry;
return registry;
}
const _REGISTRY = getGlobalRegistry();
export function register(name?: string) {
return _REGISTRY.register(name);
}
/** @ignore */
export function registerBuiltIn<
T extends EmbeddingFunctionConstructor = EmbeddingFunctionConstructor,
>(name: string, ctor: T): T {
const builtInFunctions = getBuiltInFunctions(_REGISTRY);
if (builtInFunctions.has(name)) {
return _REGISTRY.setBuiltIn(name, ctor);
}
_REGISTRY.register(name)(ctor);
builtInFunctions.add(name);
return ctor;
}
/**
* Utility function to get the global instance of the registry
* @returns `EmbeddingFunctionRegistry` The global instance of the registry
@@ -218,3 +264,52 @@ export interface EmbeddingFunctionConfig {
vectorColumn?: string;
function: EmbeddingFunction;
}
/** An [EmbeddingFunctionConfig] read back from table metadata, where the
* vector column is always recorded. */
export type ResolvedEmbeddingFunctionConfig = EmbeddingFunctionConfig & {
vectorColumn: string;
};
/** One entry of the `embedding_functions` schema metadata, with the column
* keys normalized across the bindings' spellings. */
export type EmbeddingMetadataEntry = {
name: string;
sourceColumn: string;
vectorColumn: string;
model: EmbeddingFunction["TOptions"];
};
/** The single parser for `embedding_functions` schema metadata: every reader
* goes through here, so the wire contract cannot fork between them. */
export function parseEmbeddingMetadata(json: string): EmbeddingMetadataEntry[] {
// The wire format, honestly: the Python bindings write snake_case keys.
type Raw = {
name: string;
sourceColumn?: string;
// biome-ignore lint/style/useNamingConvention: the Python wire spelling
source_column?: string;
vectorColumn?: string;
// biome-ignore lint/style/useNamingConvention: the Python wire spelling
vector_column?: string;
model: EmbeddingFunction["TOptions"];
};
const entries = <Raw[]>JSON.parse(json);
const seen = new Set<string>();
return entries.map((f) => {
const sourceColumn = f.sourceColumn ?? f.source_column;
const vectorColumn = f.vectorColumn ?? f.vector_column;
if (sourceColumn === undefined || vectorColumn === undefined) {
throw new Error(
`Embedding function "${f.name}" metadata names no source or vector column`,
);
}
if (seen.has(vectorColumn)) {
throw new Error(
`Multiple embedding configs claim vector column "${vectorColumn}"`,
);
}
seen.add(vectorColumn);
return { name: f.name, sourceColumn, vectorColumn, model: f.model };
});
}
nodejs/lancedb/embedding/transformers.ts
+3 -2
View File
@@ -3,7 +3,7 @@
import { Float, Float32 } from "../arrow";
import { EmbeddingFunction } from "./embedding_function";
import { register } from "./registry";
import { registerBuiltIn } from "./registry";
export type XenovaTransformerOptions = {
/** The wasm compatible model to use */
@@ -31,7 +31,6 @@ export type XenovaTransformerOptions = {
};
};
@register("huggingface")
export class TransformersEmbeddingFunction extends EmbeddingFunction<
string,
Partial<XenovaTransformerOptions>
@@ -158,6 +157,8 @@ export class TransformersEmbeddingFunction extends EmbeddingFunction<
}
}
registerBuiltIn("huggingface", TransformersEmbeddingFunction);
const tensorDiv = (
src: import("@huggingface/transformers").Tensor,
divBy: number,
nodejs/lancedb/header.ts
+253
View File
@@ -0,0 +1,253 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
/**
* Header providers for LanceDB remote connections.
*
* This module provides a flexible header management framework for LanceDB remote
* connections, allowing users to implement custom header strategies for
* authentication, request tracking, custom metadata, or any other header-based
* requirements.
*
* @module header
*/
/**
* Abstract base class for providing custom headers for each request.
*
* Users can implement this interface to provide dynamic headers for various purposes
* such as authentication (OAuth tokens, API keys), request tracking (correlation IDs),
* custom metadata, or any other header-based requirements. The provider is called
* before each request to ensure fresh header values are always used.
*
* @example
* Simple JWT token provider:
* ```typescript
* class JWTProvider extends HeaderProvider {
* constructor(private token: string) {
* super();
* }
*
* getHeaders(): Record<string, string> {
* return { authorization: `Bearer ${this.token}` };
* }
* }
* ```
*
* @example
* Provider with request tracking:
* ```typescript
* class RequestTrackingProvider extends HeaderProvider {
* constructor(private sessionId: string) {
* super();
* }
*
* getHeaders(): Record<string, string> {
* return {
* "X-Session-Id": this.sessionId,
* "X-Request-Id": `req-${Date.now()}`
* };
* }
* }
* ```
*/
export abstract class HeaderProvider {
/**
* Get the latest headers to be added to requests.
*
* This method is called before each request to the remote LanceDB server.
* Implementations should return headers that will be merged with existing headers.
*
* @returns Dictionary of header names to values to add to the request.
* @throws If unable to fetch headers, the exception will be propagated and the request will fail.
*/
abstract getHeaders(): Record<string, string>;
}
/**
* Example implementation: A simple header provider that returns static headers.
*
* This is an example implementation showing how to create a HeaderProvider
* for cases where headers don't change during the session.
*
* @example
* ```typescript
* const provider = new StaticHeaderProvider({
* authorization: "Bearer my-token",
* "X-Custom-Header": "custom-value"
* });
* const headers = provider.getHeaders();
* // Returns: {authorization: 'Bearer my-token', 'X-Custom-Header': 'custom-value'}
* ```
*/
export class StaticHeaderProvider extends HeaderProvider {
private _headers: Record<string, string>;
/**
* Initialize with static headers.
* @param headers - Headers to return for every request.
*/
constructor(headers: Record<string, string>) {
super();
this._headers = { ...headers };
}
/**
* Return the static headers.
* @returns Copy of the static headers.
*/
getHeaders(): Record<string, string> {
return { ...this._headers };
}
}
/**
* Token response from OAuth provider.
* @public
*/
export interface TokenResponse {
accessToken: string;
expiresIn?: number;
}
/**
* Example implementation: OAuth token provider with automatic refresh.
*
* This is an example implementation showing how to manage OAuth tokens
* with automatic refresh when they expire.
*
* @example
* ```typescript
* async function fetchToken(): Promise<TokenResponse> {
* const response = await fetch("https://oauth.example.com/token", {
* method: "POST",
* body: JSON.stringify({
* grant_type: "client_credentials",
* client_id: "your-client-id",
* client_secret: "your-client-secret"
* }),
* headers: { "Content-Type": "application/json" }
* });
* const data = await response.json();
* return {
* accessToken: data.access_token,
* expiresIn: data.expires_in
* };
* }
*
* const provider = new OAuthHeaderProvider(fetchToken);
* const headers = provider.getHeaders();
* // Returns: {"authorization": "Bearer <your-token>"}
* ```
*/
export class OAuthHeaderProvider extends HeaderProvider {
private _tokenFetcher: () => Promise<TokenResponse> | TokenResponse;
private _refreshBufferSeconds: number;
private _currentToken: string | null = null;
private _tokenExpiresAt: number | null = null;
private _refreshPromise: Promise<void> | null = null;
/**
* Initialize the OAuth provider.
* @param tokenFetcher - Function to fetch new tokens. Should return object with 'accessToken' and optionally 'expiresIn'.
* @param refreshBufferSeconds - Seconds before expiry to refresh token. Default 300 (5 minutes).
*/
constructor(
tokenFetcher: () => Promise<TokenResponse> | TokenResponse,
refreshBufferSeconds: number = 300,
) {
super();
this._tokenFetcher = tokenFetcher;
this._refreshBufferSeconds = refreshBufferSeconds;
}
/**
* Check if token needs refresh.
*/
private _needsRefresh(): boolean {
if (this._currentToken === null) {
return true;
}
if (this._tokenExpiresAt === null) {
// No expiration info, assume token is valid
return false;
}
// Refresh if we're within the buffer time of expiration
const now = Date.now() / 1000;
return now >= this._tokenExpiresAt - this._refreshBufferSeconds;
}
/**
* Refresh the token if it's expired or close to expiring.
*/
private async _refreshTokenIfNeeded(): Promise<void> {
if (!this._needsRefresh()) {
return;
}
// If refresh is already in progress, wait for it
if (this._refreshPromise) {
await this._refreshPromise;
return;
}
// Start refresh
this._refreshPromise = (async () => {
try {
const tokenData = await this._tokenFetcher();
this._currentToken = tokenData.accessToken;
if (!this._currentToken) {
throw new Error("Token fetcher did not return 'accessToken'");
}
// Set expiration if provided
if (tokenData.expiresIn) {
this._tokenExpiresAt = Date.now() / 1000 + tokenData.expiresIn;
} else {
// Token doesn't expire or expiration unknown
this._tokenExpiresAt = null;
}
} finally {
this._refreshPromise = null;
}
})();
await this._refreshPromise;
}
/**
* Get OAuth headers, refreshing token if needed.
* Note: This is synchronous for now as the Rust implementation expects sync.
* In a real implementation, this would need to handle async properly.
* @returns Headers with Bearer token authorization.
* @throws If unable to fetch or refresh token.
*/
getHeaders(): Record<string, string> {
// For simplicity in this example, we assume the token is already fetched
// In a real implementation, this would need to handle the async nature properly
if (!this._currentToken && !this._refreshPromise) {
// Synchronously trigger refresh - this is a limitation of the current implementation
throw new Error(
"Token not initialized. Call refreshToken() first or use async initialization.",
);
}
if (!this._currentToken) {
throw new Error("Failed to obtain OAuth token");
}
return { authorization: `Bearer ${this._currentToken}` };
}
/**
* Manually refresh the token.
* Call this before using getHeaders() to ensure token is available.
*/
async refreshToken(): Promise<void> {
this._currentToken = null; // Force refresh
await this._refreshTokenIfNeeded();
}
}
nodejs/lancedb/index.ts
+462 -4
View File
@@ -8,19 +8,42 @@ import {
} from "./connection";
import {
ConnectNamespaceOptions,
ConnectionOptions,
Connection as LanceDbConnection,
JsHeaderProvider as NativeJsHeaderProvider,
Session,
tokenize as nativeTokenize,
} from "./native.js";
import { HeaderProvider } from "./header";
import type { BaseTokenizer } from "./indices";
import type { FtsToken } from "./table";
// Re-export native header provider for use with connectWithHeaderProvider
export {
MaterializedView,
MaterializedViewDefinition,
MaterializedViewSelect,
} from "./materialized_view";
export { ViewDescription } from "./view";
export { JsHeaderProvider as NativeJsHeaderProvider } from "./native.js";
// OpenTelemetry metrics bridge. Only the high-level entry point is public; the
// underlying recorder/catalog/snapshot functions remain internal plumbing that
// `otel.ts` consumes from the native module.
export { instrumentLanceDbMetrics } from "./otel";
export {
AddColumnsSql,
ConnectionOptions,
ConnectNamespaceOptions,
IndexStatistics,
IndexConfig,
ClientConfig,
TimeoutConfig,
RetryConfig,
TlsConfig,
OptimizeStats,
CompactionStats,
RemovalStats,
@@ -29,38 +52,79 @@ export {
FragmentSummaryStats,
Tags,
TagContents,
BranchContents,
MergeResult,
AddResult,
AddColumnsResult,
RefreshColumnResult,
FunctionErrors,
FunctionErrorsOptions,
FunctionErrorRecord,
FunctionErrorFragment,
RefreshMaterializedViewResult,
AlterColumnsResult,
UpdateFieldMetadataResult,
DeleteResult,
DropColumnsResult,
UpdateResult,
SplitCalculatedOptions,
SplitRandomOptions,
SplitHashOptions,
SplitSequentialOptions,
ShuffleOptions,
OAuthConfig as NativeOAuthConfig,
} from "./native.js";
export {
makeArrowTable,
makeJsonField,
MakeArrowTableOptions,
Data,
VectorColumnOptions,
} from "./arrow";
export { blob, isBlobField, BlobFile } from "./blob";
export type {
BlobData,
BlobInput,
BlobOptions,
BlobRange,
BlobReadOptions,
BlobUri,
} from "./blob";
export {
Connection,
CreateTableOptions,
TableNamesOptions,
ListTablesOptions,
OpenTableOptions,
ListNamespacesOptions,
CreateNamespaceOptions,
DropNamespaceOptions,
ListNamespacesResponse,
ListTablesResponse,
CreateNamespaceResponse,
DropNamespaceResponse,
DescribeNamespaceResponse,
RenameTableOptions,
} from "./connection";
export { Session } from "./native.js";
export { JobFailureInfo, JobInfo, Session } from "./native.js";
export { Job, JobEventsOptions } from "./job";
export {
AutoQuery,
ExecutableQuery,
Query,
QueryBase,
VectorQuery,
TakeQuery,
AnalyzePlanDistributedMetrics,
BlobMode,
QueryExecutionOptions,
ColumnOrdering,
FullTextSearchOptions,
RecordBatchIterator,
FullTextQuery,
@@ -78,28 +142,68 @@ export {
Index,
IndexOptions,
IvfPqOptions,
IvfRqOptions,
IvfFlatOptions,
HnswPqOptions,
HnswSqOptions,
FtsOptions,
BaseTokenizer,
} from "./indices";
export {
Table,
Branches,
BranchColumnSummary,
BranchColumnChange,
BranchIndexSummary,
BranchRowCountSummary,
CherryPickError,
BranchDiff,
CherryPickPreview,
CherryPickResult,
AddDataOptions,
UpdateOptions,
OptimizeOptions,
Version,
WriteProgress,
FtsToken,
TokenizeTableOptions,
RTEWriteSpec,
RTEStats,
TableShardStats,
SsTableStats,
MemtableStats,
ColumnAlteration,
FieldMetadataUpdate,
} from "./table";
export {
HeaderProvider,
StaticHeaderProvider,
OAuthHeaderProvider,
TokenResponse,
} from "./header";
export {
ClientAuthMethod,
OAuthConfig,
OAuthFlowType,
OAuthSession,
SessionLogout,
SessionStatus,
TokenCacheOptions,
} from "./oauth";
export { MergeInsertBuilder, WriteExecutionOptions } from "./merge";
export * as embedding from "./embedding";
export { permutationBuilder, PermutationBuilder } from "./permutation";
export { Scannable, ScannableOptions } from "./scannable";
export * as rerankers from "./rerankers";
export {
SchemaLike,
TableLike,
DataTypeLike,
FieldLike,
RecordBatchLike,
DataLike,
@@ -108,6 +212,79 @@ export {
} from "./arrow";
export { IntoSql, packBits } from "./util";
/**
* Options for tokenizing a full-text search query without a table index.
*/
export interface TokenizeOptions {
/**
* The tokenizer to use. The default is "simple".
*/
baseTokenizer?: BaseTokenizer;
/** Language for stemming and stop words. */
language?: string;
/** Maximum token length; tokens longer than this are ignored. */
maxTokenLength?: number;
/** Whether to lowercase tokens. */
lowercase?: boolean;
/** Whether to stem tokens. */
stem?: boolean;
/** Whether to remove stop words. */
removeStopWords?: boolean;
/**
* Custom stop words that replace the built-in list for `language`.
*
* This option only affects tokenization when `removeStopWords` is true.
*
* `undefined` keeps the built-in language list. An empty array explicitly
* replaces it with no stop words.
*/
customStopWords?: string[];
/** Whether to fold ASCII characters. */
asciiFolding?: boolean;
/** N-gram minimum length. */
ngramMinLength?: number;
/** N-gram maximum length. */
ngramMaxLength?: number;
/** Whether to only emit token prefixes for the n-gram tokenizer. */
prefixOnly?: boolean;
}
/**
* Tokenize a full-text search query using an explicit tokenizer.
*
* This does not require a table or FTS index. The tokenizer options match
* {@link Index.fts}.
*/
export async function tokenize(
query: string,
options?: Partial<TokenizeOptions>,
): Promise<FtsToken[]> {
return await nativeTokenize(
query,
options?.baseTokenizer,
options?.language,
options?.maxTokenLength,
options?.lowercase,
options?.stem,
options?.removeStopWords,
options?.customStopWords,
options?.asciiFolding,
options?.ngramMinLength,
options?.ngramMaxLength,
options?.prefixOnly,
);
}
/**
* Connect to a LanceDB instance at the given URI.
*
@@ -131,11 +308,27 @@ export { IntoSql, packBits } from "./util";
* {storageOptions: {timeout: "60s"}
* });
* ```
* @example
* Using with a header provider for per-request authentication:
* ```ts
* const provider = new StaticHeaderProvider({
* "X-API-Key": "my-key"
* });
* const conn = await connectWithHeaderProvider(
* "db://host:port",
* options,
* provider
* );
* ```
*/
export async function connect(
uri: string,
options?: Partial<ConnectionOptions>,
session?: Session,
headerProvider?:
| HeaderProvider
| (() => Record<string, string>)
| (() => Promise<Record<string, string>>),
): Promise<Connection>;
/**
* Connect to a LanceDB instance at the given URI.
@@ -169,18 +362,58 @@ export async function connect(
): Promise<Connection>;
export async function connect(
uriOrOptions: string | (Partial<ConnectionOptions> & { uri: string }),
options?: Partial<ConnectionOptions>,
optionsOrSession?: Partial<ConnectionOptions> | Session,
sessionOrHeaderProvider?:
| Session
| HeaderProvider
| (() => Record<string, string>)
| (() => Promise<Record<string, string>>),
headerProvider?:
| HeaderProvider
| (() => Record<string, string>)
| (() => Promise<Record<string, string>>),
): Promise<Connection> {
let uri: string | undefined;
let finalOptions: Partial<ConnectionOptions> = {};
let finalHeaderProvider:
| HeaderProvider
| (() => Record<string, string>)
| (() => Promise<Record<string, string>>)
| undefined;
if (typeof uriOrOptions !== "string") {
// First overload: connect(options)
const { uri: uri_, ...opts } = uriOrOptions;
uri = uri_;
finalOptions = opts;
} else {
// Second overload: connect(uri, options?, session?, headerProvider?)
uri = uriOrOptions;
finalOptions = options || {};
// Handle optionsOrSession parameter
if (optionsOrSession && "inner" in optionsOrSession) {
// Second param is session, so no options provided
finalOptions = {};
} else {
// Second param is options
finalOptions = (optionsOrSession as Partial<ConnectionOptions>) || {};
}
// Handle sessionOrHeaderProvider parameter
if (
sessionOrHeaderProvider &&
(typeof sessionOrHeaderProvider === "function" ||
"getHeaders" in sessionOrHeaderProvider)
) {
// Third param is header provider
finalHeaderProvider = sessionOrHeaderProvider as
| HeaderProvider
| (() => Record<string, string>)
| (() => Promise<Record<string, string>>);
} else {
// Third param is session, header provider is fourth param
finalHeaderProvider = headerProvider;
}
}
if (!uri) {
@@ -191,6 +424,231 @@ export async function connect(
(<ConnectionOptions>finalOptions).storageOptions = cleanseStorageOptions(
(<ConnectionOptions>finalOptions).storageOptions,
);
const nativeConn = await LanceDbConnection.new(uri, finalOptions);
// Create native header provider if one was provided
let nativeProvider: NativeJsHeaderProvider | undefined;
if (finalHeaderProvider) {
if (typeof finalHeaderProvider === "function") {
nativeProvider = new NativeJsHeaderProvider(async () =>
finalHeaderProvider(),
);
} else if (
finalHeaderProvider &&
typeof finalHeaderProvider.getHeaders === "function"
) {
nativeProvider = new NativeJsHeaderProvider(async () =>
finalHeaderProvider.getHeaders(),
);
}
}
const nativeConn = await LanceDbConnection.new(
uri,
finalOptions,
nativeProvider,
);
return new LocalConnection(nativeConn);
}
/**
* Configuration for the built-in directory namespace (`"dir"`).
*
* The directory namespace stores tables under a single root path (local
* filesystem or object storage URI). See
* {@link https://docs.lancedb.com/namespaces} for the documented surface;
* less-common knobs live under {@link DirNamespaceConfig.extraProperties}.
*/
export interface DirNamespaceConfig {
/** Root path or URI containing the LanceDB tables. */
root: string;
/**
* Whether to maintain a namespace manifest at the root. Required for
* child namespaces. Defaults to true on the impl side.
*/
manifestEnabled?: boolean;
/**
* Additional raw properties passed verbatim to the namespace
* implementation (e.g. `storage.*`, `credential_vendor.*`). Typed
* fields above take precedence on key collision.
*/
extraProperties?: Record<string, string>;
}
/**
* Configuration for the built-in REST namespace (`"rest"`).
*
* The REST namespace talks to a remote catalog server over HTTP. See
* {@link https://docs.lancedb.com/namespaces} for the documented surface;
* less-common knobs (TLS, metrics) live under
* {@link RestNamespaceConfig.extraProperties}.
*/
export interface RestNamespaceConfig {
/** Catalog endpoint URL. */
uri: string;
/**
* HTTP headers forwarded with each request. Keys are passed through
* as-is (e.g. `"x-api-key"`, `"Authorization"`).
*/
headers?: Record<string, string>;
/**
* Additional raw properties passed verbatim to the namespace
* implementation (e.g. `tls.*`, `ops_metrics_enabled`, `delimiter`).
* Typed fields above take precedence on key collision.
*/
extraProperties?: Record<string, string>;
}
function dirConfigToProperties(
config: DirNamespaceConfig,
): Record<string, string> {
// Spread the whole input so that unknown keys (e.g. a raw `manifest_enabled`
// passed via the dynamic-impl path) flow through instead of being dropped.
// Typed transformations layer on top.
const { manifestEnabled, extraProperties, ...rest } = config;
const properties: Record<string, string> = {
...(extraProperties ?? {}),
...(rest as Record<string, string>),
};
if (manifestEnabled !== undefined) {
properties.manifest_enabled = String(manifestEnabled);
}
return properties;
}
function restConfigToProperties(
config: RestNamespaceConfig,
): Record<string, string> {
const { headers, extraProperties, ...rest } = config;
const properties: Record<string, string> = {
...(extraProperties ?? {}),
...(rest as Record<string, string>),
};
if (headers) {
for (const [name, value] of Object.entries(headers)) {
properties[`headers.${name}`] = value;
}
}
return properties;
}
/**
* Connect to a LanceDB database through a namespace.
*
* Unlike {@link connect}, which routes by URI scheme (local path vs.
* `db://` cloud), `connectNamespace` always returns a namespace-backed
* connection. The `implName` selects the namespace implementation:
*
* - `"dir"` — directory namespace, configured with {@link DirNamespaceConfig}.
* - `"rest"` — remote REST catalog, configured with {@link RestNamespaceConfig}.
* - Any other string — full module path for a custom implementation,
* configured with a free-form string-keyed `properties` map.
*
* @example Typed dir namespace
* ```ts
* const db = await connectNamespace("dir", { root: "/path/to/db" });
* await db.createTable("users", [{ id: 1 }]);
* ```
*
* @example Typed REST namespace with auth headers
* ```ts
* const db = await connectNamespace("rest", {
* uri: "https://catalog.example.com",
* headers: { "x-api-key": process.env.CATALOG_KEY ?? "" },
* });
* ```
*
* @example Custom implementation with raw properties
* ```ts
* const db = await connectNamespace("my.custom.Namespace", {
* endpoint: "...",
* });
* ```
*/
export function connectNamespace(
implName: "dir",
config: DirNamespaceConfig,
options?: Partial<ConnectNamespaceOptions>,
): Promise<Connection>;
/**
* Connect through the built-in REST namespace.
*
* Configured with {@link RestNamespaceConfig}. See the function-level
* documentation above for the full surface, examples, and how this
* relates to {@link connect}.
*
* @example
* ```ts
* const db = await connectNamespace("rest", {
* uri: "https://catalog.example.com",
* headers: { "x-api-key": process.env.CATALOG_KEY ?? "" },
* });
* ```
*/
export function connectNamespace(
implName: "rest",
config: RestNamespaceConfig,
options?: Partial<ConnectNamespaceOptions>,
): Promise<Connection>;
/**
* Connect through a custom namespace implementation by full module path,
* configured with a free-form string-keyed `properties` map. Use the
* typed overloads above for the built-in `"dir"` and `"rest"` impls.
*
* See the function-level documentation above for examples and how this
* relates to {@link connect}.
*
* @example
* ```ts
* const db = await connectNamespace("my.custom.Namespace", {
* endpoint: "...",
* });
* ```
*/
export function connectNamespace(
implName: string,
properties: Record<string, string>,
options?: Partial<ConnectNamespaceOptions>,
): Promise<Connection>;
export async function connectNamespace(
implName: string,
configOrProperties:
| DirNamespaceConfig
| RestNamespaceConfig
| Record<string, string>,
options?: Partial<ConnectNamespaceOptions>,
): Promise<Connection> {
let properties: Record<string, string>;
if (implName === "dir") {
properties = dirConfigToProperties(
configOrProperties as DirNamespaceConfig,
);
} else if (implName === "rest") {
properties = restConfigToProperties(
configOrProperties as RestNamespaceConfig,
);
} else {
properties = configOrProperties as Record<string, string>;
}
const finalOptions: ConnectNamespaceOptions = (options ??
{}) as ConnectNamespaceOptions;
finalOptions.storageOptions = cleanseStorageOptions(
finalOptions.storageOptions,
);
const nativeConn = await LanceDbConnection.newWithNamespace(
implName,
properties,
finalOptions,
);
return new LocalConnection(nativeConn);
}
export {
Catalog,
CatalogOptions,
DatabaseNames,
connectCatalog,
} from "./catalog";
export { Listing, ListingOptions } from "./listing";
nodejs/lancedb/indices.ts
+152 -4
View File
@@ -26,7 +26,7 @@ export interface IvfPqOptions {
* This value controls how much the vector is compressed during the quantization step.
* The more sub vectors there are the less the vector is compressed. The default is
* the dimension of the vector divided by 16. If the dimension is not evenly divisible
* by 16 we use the dimension divded by 8.
* by 16 we use the dimension divided by 8.
*
* The above two cases are highly preferred. Having 8 or 16 values per subvector allows
* us to use efficient SIMD instructions.
@@ -112,6 +112,77 @@ export interface IvfPqOptions {
sampleRate?: number;
}
export interface IvfRqOptions {
/**
* The number of IVF partitions to create.
*
* This value should generally scale with the number of rows in the dataset.
* By default the number of partitions is the square root of the number of
* rows.
*
* If this value is too large then the first part of the search (picking the
* right partition) will be slow. If this value is too small then the second
* part of the search (searching within a partition) will be slow.
*/
numPartitions?: number;
/**
* Number of bits per dimension for residual quantization.
*
* This value controls how much each residual component is compressed. The more
* bits, the more accurate the index will be but the slower search. Typical values
* are small integers; the default is 1 bit per dimension.
*/
numBits?: number;
/**
* Distance type to use to build the index.
*
* Default value is "l2".
*
* This is used when training the index to calculate the IVF partitions
* (vectors are grouped in partitions with similar vectors according to this
* distance type) and during quantization.
*
* The distance type used to train an index MUST match the distance type used
* to search the index. Failure to do so will yield inaccurate results.
*
* The following distance types are available:
*
* "l2" - Euclidean distance.
* "cosine" - Cosine distance.
* "dot" - Dot product.
*/
distanceType?: "l2" | "cosine" | "dot";
/**
* Max iterations to train IVF kmeans.
*
* When training an IVF index we use kmeans to calculate the partitions. This parameter
* controls how many iterations of kmeans to run.
*
* The default value is 50.
*/
maxIterations?: number;
/**
* The number of vectors, per partition, to sample when training IVF kmeans.
*
* When an IVF index is trained, we need to calculate partitions. These are groups
* of vectors that are similar to each other. To do this we use an algorithm called kmeans.
*
* Running kmeans on a large dataset can be slow. To speed this up we run kmeans on a
* random sample of the data. This parameter controls the size of the sample. The total
* number of vectors used to train the index is `sample_rate * num_partitions`.
*
* Increasing this value might improve the quality of the index but in most cases the
* default should be sufficient.
*
* The default value is 256.
*/
sampleRate?: number;
}
/**
* Options to create an `HNSW_PQ` index
*/
@@ -157,7 +228,7 @@ export interface HnswPqOptions {
* This value controls how much the vector is compressed during the quantization step.
* The more sub vectors there are the less the vector is compressed. The default is
* the dimension of the vector divided by 16. If the dimension is not evenly divisible
* by 16 we use the dimension divded by 8.
* by 16 we use the dimension divided by 8.
*
* The above two cases are highly preferred. Having 8 or 16 values per subvector allows
* us to use efficient SIMD instructions.
@@ -415,6 +486,17 @@ export interface IvfFlatOptions {
sampleRate?: number;
}
export type BaseTokenizer =
| "simple"
| "whitespace"
| "raw"
| "ngram"
| "code"
| "icu"
| "icu/split"
| `jieba/${string}`
| `lindera/${string}`;
/**
* Options to create a full text search index
*/
@@ -438,8 +520,14 @@ export interface FtsOptions {
* "whitespace" - Whitespace tokenizer. This tokenizer splits the text into tokens using whitespace as a delimiter.
*
* "raw" - Raw tokenizer. This tokenizer does not split the text into tokens and indexes the entire text as a single token.
*
* "code" - Code tokenizer. This tokenizer keeps snake_case and camelCase identifiers as single tokens.
*
* "icu" - ICU dictionary-based word segmentation.
*
* "icu/split" - ICU segmentation with simple-style delimiter splitting.
*/
baseTokenizer?: "simple" | "whitespace" | "raw" | "ngram";
baseTokenizer?: BaseTokenizer;
/**
* language for stemming and stop words
@@ -468,6 +556,16 @@ export interface FtsOptions {
*/
removeStopWords?: boolean;
/**
* Custom stop words that replace the built-in list for `language`.
*
* This option only affects tokenization when `removeStopWords` is true.
*
* `undefined` keeps the built-in language list. An empty array explicitly
* replaces it with no stop words.
*/
customStopWords?: string[];
/**
* whether to remove punctuation
*/
@@ -487,6 +585,14 @@ export interface FtsOptions {
* whether to only index the prefix of the token for ngram tokenizer
*/
prefixOnly?: boolean;
/**
* Number of documents per compressed posting block.
*
* The default is 128. Supported values are 128 and 256. A value of 256 uses
* the experimental FTS V3 format and may introduce breaking changes.
*/
blockSize?: 128 | 256;
}
export class Index {
@@ -523,6 +629,35 @@ export class Index {
options?.distanceType,
options?.numPartitions,
options?.numSubVectors,
options?.numBits,
options?.maxIterations,
options?.sampleRate,
),
);
}
/**
* Create an IvfRq index
*
* IVF-RQ (RabitQ Quantization) compresses vectors using RabitQ quantization
* and organizes them into IVF partitions.
*
* The compression scheme is called RabitQ quantization. Each dimension is quantized into a small number of bits.
* The parameters `num_bits` and `num_partitions` control this process, providing a tradeoff
* between index size (and thus search speed) and index accuracy.
*
* The partitioning process is called IVF and the `num_partitions` parameter controls how
* many groups to create.
*
* Note that training an IVF RQ index on a large dataset is a slow operation and
* currently is also a memory intensive operation.
*/
static ivfRq(options?: Partial<IvfRqOptions>) {
return new Index(
LanceDbIndex.ivfRq(
options?.distanceType,
options?.numPartitions,
options?.numBits,
options?.maxIterations,
options?.sampleRate,
),
@@ -602,6 +737,17 @@ export class Index {
return new Index(LanceDbIndex.labelList());
}
/**
* Create an FM-Index.
*
* An FM-Index is a scalar index on string or binary columns that accelerates
* substring search, i.e. `contains(col, 'needle')`. Unlike the tokenized
* full-text-search index, it matches arbitrary substrings of the raw bytes.
*/
static fm() {
return new Index(LanceDbIndex.fm());
}
/**
* Create a full text search index
*
@@ -622,10 +768,12 @@ export class Index {
options?.lowercase,
options?.stem,
options?.removeStopWords,
options?.customStopWords,
options?.asciiFolding,
options?.ngramMinLength,
options?.ngramMaxLength,
options?.prefixOnly,
options?.blockSize,
),
);
}
@@ -680,7 +828,7 @@ export interface IndexOptions {
/**
* Advanced index configuration
*
* This option allows you to specify a specfic index to create and also
* This option allows you to specify a specific index to create and also
* allows you to pass in configuration for training the index.
*
* See the static methods on Index for details on the various index types.
nodejs/lancedb/job.ts
+188
View File
@@ -0,0 +1,188 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { Table as ArrowTable, tableFromIPC } from "apache-arrow";
import { JobFailureInfo, Job as NativeJob } from "./native";
/** Which of a job's events {@link Job.events} returns. */
export interface JobEventsOptions {
/** Maximum event rows to return, up to the server maximum of 10,000. */
limit?: number;
/** SQL-like filter over the event columns. */
filter?: string;
}
/**
* A handle to an operation that may still be running.
*
* The operation may already be complete when the handle is created.
*
* The detail getters read what the handle last observed. Submitting an
* operation returns only a job id, so populating them eagerly would cost an
* extra round trip on every call:
*
* - {@link Job.refresh} and {@link Job.status} fetch the whole record.
* - {@link Job.wait} records the terminal state it establishes, but not the
* rest of the record.
* - Everything is null until one of those runs.
*
* @hideconstructor
*/
export class Job {
private readonly inner: NativeJob;
constructor(inner: NativeJob) {
this.inner = inner;
}
/**
* Identifies the operation on the server that is running it.
*
* Operations that run in this process have no server id. The value is
* opaque: parsing it or storing it to resume the job later is not supported.
*/
get id(): string | null {
return this.inner.id ?? null;
}
/** The last observed lifecycle state, without contacting the backend. */
get state(): string | null {
return this.inner.state ?? null;
}
/**
* The job's type, as the server names it. Null for an in-process job, which
* has no server-side record.
*/
get jobType(): string | null {
return this.inner.jobType ?? null;
}
/** When the job was created, in milliseconds since the epoch. */
get creationMs(): number | null {
return this.inner.creationMs ?? null;
}
/** The job-type-specific specification it was submitted with. */
// biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
get spec(): any | null {
return parseJson(this.inner.specJson);
}
/**
* The job-type-specific terminal result. Null until the job succeeds, so a
* job that never terminates reports its progress through {@link Job.events}
* instead.
*/
// biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
get result(): any | null {
return parseJson(this.inner.resultJson);
}
/** Why the job failed, when it failed and the server reports a reason. */
get failure(): JobFailureInfo | null {
return this.inner.failure ?? null;
}
/**
* The operation's current lifecycle state: "running", "finished", "failed",
* or "cancelled".
*
* A point snapshot; unlike {@link Job.wait} it does not block or reject on a
* terminal failure state. Also refreshes the getters above.
*/
async status(): Promise<string> {
return this.inner.status();
}
/** Wait until the operation reaches a terminal state. */
async wait(): Promise<void> {
return this.inner.wait();
}
/** Request cancellation. Cancelling a finished operation is a no-op. */
async cancel(): Promise<void> {
return this.inner.cancel();
}
/**
* Ask the backend for this job's current state, and for a server-side job
* its full record, then cache it for the getters above.
*/
async refresh(): Promise<void> {
return this.inner.refresh();
}
/**
* This job's recorded lifecycle events.
*
* Where the getters above report a terminal result only once the job reaches
* one, events are written as the job runs and outlive the workers that
* produced them. A distributed job records a `claim`/`claim_complete` pair
* per unit of work, each carrying `rows_processed`, so a job that never
* finishes still accounts for what it did.
*
* The server caps results at 1000 rows by default and 10,000 at most, and
* truncates without saying so, so pass `limit` for a job that emits an event
* per fragment. `filter` is a SQL-like expression over the `state`,
* `updated_by`, `emitted_from`, `emitted_by`, and `claim_entity` columns.
*/
async events(options?: JobEventsOptions): Promise<ArrowTable> {
const buf = await this.inner.events(options?.limit, options?.filter);
if (buf.length === 0) {
return new ArrowTable();
}
return tableFromIPC(buf);
}
/**
* Every field the handle currently knows, one per line, with the JSON
* payloads indented -- a refresh job's spec and result are the point of
* printing it.
*/
toString(): string {
if (this.state === null) {
const known = this.id === null ? "" : `id=${JSON.stringify(this.id)}, `;
return `Job(${known}not refreshed)`;
}
const fields: string[] = [];
if (this.id !== null) {
fields.push(`id=${JSON.stringify(this.id)}`);
}
fields.push(`state=${JSON.stringify(this.state)}`);
if (this.jobType !== null) {
fields.push(`jobType=${JSON.stringify(this.jobType)}`);
}
if (this.creationMs !== null) {
fields.push(`creationMs=${this.creationMs}`);
}
for (const [name, value] of [
["spec", this.spec],
["result", this.result],
] as const) {
if (value !== null) {
fields.push(`${name}=${indentJson(value)}`);
}
}
if (this.failure !== null) {
fields.push(`failure=${indentJson(this.failure)}`);
}
return `Job(${fields.map((field) => `\n${REPR_INDENT}${field},`).join("")}\n)`;
}
[Symbol.for("nodejs.util.inspect.custom")](): string {
return this.toString();
}
}
const REPR_INDENT = " ";
// biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
function indentJson(value: any): string {
return JSON.stringify(value, null, 4).replace(/\n/g, `\n${REPR_INDENT}`);
}
// biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
function parseJson(raw: string | null | undefined): any | null {
return raw === null || raw === undefined ? null : JSON.parse(raw);
}
nodejs/lancedb/listing.ts
+73
View File
@@ -0,0 +1,73 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
/** Pagination controls for lazy resource listings. */
export interface ListingOptions {
/** Starting continuation token; omitted starts at the beginning. */
pageToken?: string;
/** Maximum results per request, not a limit on the whole listing. */
pageLimit?: number;
}
/** @internal */
export function validateListingOptions(options: ListingOptions): void {
if (
options.pageLimit !== undefined &&
(!Number.isInteger(options.pageLimit) ||
options.pageLimit <= 0 ||
options.pageLimit > 2147483647)
) {
throw new Error(
"Listing page limit must be an integer between 1 and 2147483647",
);
}
}
/**
* A lazy async iterator over database resources with pagination state.
* Requests occur during iteration; request errors terminate the iterator.
*
* @example
* ```ts
* for await (const name of connection.listViews([], { pageLimit: 20 })) {
* console.log(name);
* }
* ```
*/
export class Listing<T> implements AsyncIterableIterator<T> {
/** @hidden */
constructor(
private readonly inner: {
next(): Promise<T | null | undefined>;
numPageResults(): number;
pageToken(): string | null | undefined;
},
) {}
[Symbol.asyncIterator](): AsyncIterableIterator<T> {
return this;
}
/** Fetch the next item, requesting another page only when needed. */
async next(): Promise<IteratorResult<T>> {
const value = await this.inner.next();
return value == null
? { done: true, value: undefined }
: { done: false, value };
}
/** Number of cached items available without another request. */
numPageResults(): number {
return this.inner.numPageResults();
}
/**
* Token for the next request. Initially this is the supplied starting token.
* Returns undefined after fetching the final page, even with cached items.
* Drain the cache before saving a token to avoid skipping items on resumption.
* Failed requests terminate iteration and retain their token for a new iterator.
*/
pageToken(): string | undefined {
return this.inner.pageToken() ?? undefined;
}
}
nodejs/lancedb/materialized_view.ts
+200
View File
@@ -0,0 +1,200 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { RefreshMaterializedViewResult } from "./native";
import { Table } from "./table";
/** Schema metadata key holding a materialized view's definition. */
export const DEFINITION_META_KEY = "mv.definition";
/**
* The newest stored layout this version reads: `{"format": N, "query": "<SQL>"}`,
* format 2 being a query with `GROUP BY`.
*/
export const DEFINITION_FORMAT = 2;
/**
* The query that defines a materialized view, as stored:
* `SELECT columns FROM [ns.]table [, function(args) AS alias | , UNNEST(column) AS alias]
* [WHERE predicate] [GROUP BY expr, ...] [LIMIT n]`. A Function in `FROM`
* position yields one row per element it returns.
*/
export interface MaterializedViewDefinition {
/** The defining query, in the canonical spelling the server stores. */
query: string;
}
/**
* The view's columns: column names, `[alias, SQL expression]` pairs, or a
* record of the same. A bare name projects itself.
*/
export type MaterializedViewSelect =
| (string | [string, string])[]
| Record<string, string>;
/**
* @internal Reject a numeric option N-API would otherwise silently coerce:
* `Infinity` reaches Rust as 0, `1.5` as 1.
*/
export function validateNonNegativeInteger(
value: number | undefined,
name: string,
): void {
if (value !== undefined && !(Number.isSafeInteger(value) && value >= 0)) {
throw new Error(`${name} must be a non-negative integer`);
}
}
/** @internal Quote a column name as a Lance SQL identifier (backticks). */
function quoteIdentifier(name: string): string {
return "`" + name.replace(/`/g, "``") + "`";
}
/**
* @internal Normalize a select argument into `[alias, expression]` pairs.
* A bare name projects itself and is quoted, so any valid column name works;
* pair and record entries are kept verbatim because their right side is an
* expression.
*/
export function normalizeSelect(
select?: MaterializedViewSelect,
): [string, string][] | undefined {
if (select === undefined) {
return undefined;
}
if (Array.isArray(select)) {
return select.map((item) =>
typeof item === "string" ? [item, quoteIdentifier(item)] : item,
);
}
return Object.entries(select);
}
/** @internal Parse a definition off a table's stored schema metadata. */
export function definitionFromMetadata(
metadata: Map<string, string>,
name: string,
): MaterializedViewDefinition {
const raw = metadata.get(DEFINITION_META_KEY);
if (raw === undefined) {
throw new Error(`Table '${name}' is not a materialized view`);
}
return definitionFromJson(raw, name);
}
/** @internal Parse the backend-independent definition returned by native code. */
export function definitionFromJson(
raw: string,
name: string,
): MaterializedViewDefinition {
// biome-ignore lint/suspicious/noExplicitAny: raw JSON
const value: any = JSON.parse(raw);
if (value.format !== undefined) {
// A newer writer's layout is reported, never guessed at.
if (!Number.isInteger(value.format) || value.format > DEFINITION_FORMAT) {
throw new Error(
`materialized view '${name}' is stored in format ${value.format}, ` +
"which this version of lancedb cannot refresh",
);
}
return { query: value.query };
}
// The structured layout written before the format number.
if (value.kind !== "select" && value.kind !== "namespaced_select") {
throw new Error(
`materialized view '${name}' is stored in format kind '${value.kind}', ` +
"which this version of lancedb cannot refresh",
);
}
const limit = value.limit ?? undefined;
// JSON.parse rounds integers past 2^53; every exact u64 parses to a safe
// integer and every rounded one does not, so this rejects precisely the
// values a number cannot carry.
if (limit !== undefined && !Number.isSafeInteger(limit)) {
throw new Error(
`materialized view '${name}' has a stored limit too large to represent exactly`,
);
}
return { query: legacyQuery(value, limit) };
}
function legacyIdent(name: string): string {
return /^[a-z_][a-z0-9_]*$/.test(name)
? name
: `\`${name.replace(/`/g, "``")}\``;
}
/** Render the pre-format structured layout as the query it described. */
// biome-ignore lint/suspicious/noExplicitAny: raw JSON
function legacyQuery(value: any, limit: number | undefined): string {
// biome-ignore lint/suspicious/noExplicitAny: raw JSON
const projections: any[] = value.projections ?? [];
const columns =
projections.length === 0
? "*"
: projections
.map((p) =>
p.expression === p.output || p.expression === `\`${p.output}\``
? p.expression
: `${p.expression} AS ${legacyIdent(p.output)}`,
)
.join(", ");
const table = [...(value.source_namespace ?? []), value.source_table]
.map(legacyIdent)
.join(".");
let query = `SELECT ${columns} FROM ${table}`;
if (value.filter !== undefined && value.filter !== null) {
query += ` WHERE ${value.filter}`;
}
if (limit !== undefined) {
query += ` LIMIT ${limit}`;
}
return query;
}
/**
* A handle on a materialized view: its table plus its definition.
*
* Obtained from {@link Connection#createMaterializedView} or
* {@link Connection#openMaterializedView}. The view is a normal table --
* queries, indexes and search all apply through {@link MaterializedView#table}
* -- whose contents are maintained by {@link MaterializedView#refresh}.
*/
export class MaterializedView {
private readonly inner: Table;
constructor(table: Table) {
this.inner = table;
}
get name(): string {
return this.inner.name;
}
/** The view, as the table it is. */
table(): Table {
return this.inner;
}
/** The query that defines the view. */
async definition(): Promise<MaterializedViewDefinition> {
return definitionFromJson(
await this.inner.materializedViewDefinition(),
this.name,
);
}
/**
* Recompute the view from its source.
*
* Every refresh rebuilds the complete view. For local views,
* `sourceVersion` refreshes to that source version instead of the latest;
* remote SQL refreshes do not support source-version pinning.
*/
async refresh(options?: {
sourceVersion?: number;
}): Promise<RefreshMaterializedViewResult> {
validateNonNegativeInteger(options?.sourceVersion, "sourceVersion");
return await this.inner.refreshMaterializedView(options?.sourceVersion);
}
}
nodejs/lancedb/merge.ts
+49 -1
View File
@@ -27,7 +27,7 @@ export class MergeInsertBuilder {
* but that behavior is subject to change.
*
* An optional condition may be specified. If it is, then only
* matched rows that satisfy the condtion will be updated. Any
* matched rows that satisfy the condition will be updated. Any
* rows that do not satisfy the condition will be left as they
* are. Failing to satisfy the condition does not cause a
* "matched row" to become a "not matched" row.
@@ -70,6 +70,54 @@ export class MergeInsertBuilder {
this.#schema,
);
}
/**
* Controls whether to use indexes for the merge operation.
*
* When set to `true` (the default), the operation will use an index if available
* on the join key for improved performance. When set to `false`, it forces a full
* table scan even if an index exists. This can be useful for benchmarking or when
* the query optimizer chooses a suboptimal path.
*
* @param useIndex - Whether to use indices for the merge operation. Defaults to `true`.
*/
useIndex(useIndex: boolean): MergeInsertBuilder {
return new MergeInsertBuilder(
this.#native.useIndex(useIndex),
this.#schema,
);
}
/**
* Control RTE routing for this merge.
*
* By default (unset), a `mergeInsert` on a table with an RTE write spec is
* routed through Lance's MemWAL shard writer, and a table without one uses the
* standard path.
*
* @param enable - `true` forces RTE routing and errors if the table has no
* RTE write spec. `false` forces the standard write path even when a spec is set.
*/
useRTE(enable: boolean): MergeInsertBuilder {
return new MergeInsertBuilder(this.#native.useRTE(enable), this.#schema);
}
/**
* Controls how an LSM merge checks that its input targets a single shard.
*
* When a table has an RTE write spec, every row in a `mergeInsert` call must
* route to the same shard. When `true` (the default), every row is inspected
* to verify this. When `false`, only the first row is inspected and the
* shard it routes to is used for the whole input — a faster path for callers
* that have already pre-sharded their input. Has no effect on tables without
* an RTE write spec.
*
* @param validateSingleShard - Whether to check every row routes to one shard. Defaults to `true`.
*/
validateSingleShard(validateSingleShard: boolean): MergeInsertBuilder {
return new MergeInsertBuilder(
this.#native.validateSingleShard(validateSingleShard),
this.#schema,
);
}
/**
* Executes the merge insert operation
*
nodejs/lancedb/oauth.ts
+318
View File
@@ -0,0 +1,318 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import {
OAuthConfig as NativeOAuthConfig,
OAuthSession as NativeOAuthSession,
} from "./native";
/**
* OAuth authentication flow types.
*/
export enum OAuthFlowType {
/** Client Credentials grant (service-to-service / M2M). */
ClientCredentials = "client_credentials",
/** Interactive Authorization Code grant, using PKCE by default. */
AuthorizationCode = "authorization_code",
/** Device Authorization grant for CLI and headless environments. */
DeviceCode = "device_code",
/** Azure Managed Identity via IMDS. */
AzureManagedIdentity = "azure_managed_identity",
}
/**
* Options for the persistent OAuth token cache.
*
* The cache is opt-in: it is only used when set as `tokenCache` on
* {@link OAuthConfig}. Only refresh tokens are persisted, in a private
* directory with owner-only permissions, so short-lived processes can reuse
* an authenticated session instead of re-prompting on every start.
*
* Multiple identities (issuer, client, scopes, resource, audience, flow, client authentication)
* get separate cache entries. Within one identity the most recent login wins.
*/
export interface TokenCacheOptions {
/**
* Directory that holds cached credentials. Defaults to
* `$XDG_CACHE_HOME/lancedb/oauth`, `$HOME/.cache/lancedb/oauth` on Unix,
* or `%LOCALAPPDATA%\\lancedb\\oauth` on Windows. The directory is created
* with owner-only permissions (`0700`) when missing.
*/
cacheDir?: string;
/**
* How long to wait for the cross-process refresh lock before failing, in
* seconds (default: 30).
*/
lockTimeoutSecs?: number;
}
/**
* How the client authenticates to the OAuth token endpoint.
*
* The method applies to every OAuth request that carries client
* authentication: client-credentials, authorization-code exchange,
* refresh-token, and device-authorization requests. The Azure managed
* identity flow ignores this option.
*/
export enum ClientAuthMethod {
/**
* No client authentication, for public clients using PKCE or the device
* flow. Cannot be combined with `clientSecret`.
*/
None = "none",
/**
* HTTP Basic authentication. This is the RFC 6749 recommended method and
* the normal default for confidential clients, including default Okta
* applications. Requires `clientSecret`.
*/
ClientSecretBasic = "client_secret_basic",
/**
* Credentials in the request body, for providers configured to require it.
* Requires `clientSecret`.
*/
ClientSecretPost = "client_secret_post",
}
/**
* OAuth configuration for LanceDB authentication.
*
* This is the public TypeScript OAuth configuration type. The generated
* `NativeOAuthConfig` type has the same runtime shape but is an implementation
* detail of the napi-rs binding.
*
* All token acquisition and refresh is handled in the Rust layer.
* This config is passed through to Rust via napi-rs.
*
* @example Client Credentials (service-to-service):
* ```typescript
* const config: OAuthConfig = {
* issuerUrl: "https://login.microsoftonline.com/{tenant}/v2.0",
* clientId: "app-id",
* clientSecret: "secret",
* scopes: ["api://lancedb-api/.default"],
* };
* ```
*
* Providers requiring an explicit target can set `resource` and/or `audience`:
* ```typescript
* const targeted: OAuthConfig = {
* issuerUrl: "https://issuer.example.com",
* clientId: "app-id",
* clientSecret: "secret",
* scopes: ["read"],
* resource: "https://api.example.com",
* audience: "lancedb-api",
* };
* ```
*
* @example Azure Managed Identity:
* ```typescript
* const config: OAuthConfig = {
* issuerUrl: "https://login.microsoftonline.com/{tenant}/v2.0",
* clientId: "app-id",
* scopes: ["api://lancedb-api/.default"],
* flow: OAuthFlowType.AzureManagedIdentity,
* };
* ```
*
* @example Authorization Code with PKCE:
* The authorization URL is written to stderr before LanceDB tries to open a
* browser, so it can be copied in headless environments.
* ```typescript
* const config: OAuthConfig = {
* issuerUrl: "https://login.microsoftonline.com/{tenant}/v2.0",
* clientId: "app-id",
* scopes: ["openid", "api://lancedb-api/access"],
* flow: OAuthFlowType.AuthorizationCode,
* };
* ```
*
* Device Authorization writes the verification URL and user code to stderr
* before polling begins.
*/
export interface OAuthConfig {
/**
* OIDC issuer URL or OAuth authority URL.
* For Azure: `https://login.microsoftonline.com/{tenant_id}/v2.0`
*/
issuerUrl: string;
/** Application / Client ID. */
clientId: string;
/**
* OAuth scopes to request.
* For Azure managed identity, exactly one scope or resource is required.
* For example: `["api://{app_id}/.default"]`
*/
scopes: string[];
/**
* Resource indicator (RFC 8707), forwarded verbatim to authorization and token
* endpoints, including refresh requests. Must be an absolute URI without a
* fragment. Not supported for Azure managed identity.
*/
resource?: string;
/**
* Provider-specific audience, forwarded to authorization and token endpoints,
* including refresh requests. Not supported for Azure managed identity.
*/
audience?: string;
/** Authentication flow (default: ClientCredentials). */
flow?: OAuthFlowType;
/** Client secret (required for ClientCredentials). */
clientSecret?: string;
/**
* How the client authenticates to the token endpoint (default: auto).
* With a `clientSecret` the default is `ClientAuthMethod.ClientSecretBasic`,
* which matches the RFC 6749 recommendation and the default configuration
* of Okta confidential applications; without a secret the client is public
* and no client authentication is sent.
*/
clientAuthMethod?: ClientAuthMethod;
/** Loopback redirect URI for AuthorizationCode. */
redirectUri?: string;
/** Port for the AuthorizationCode loopback callback server (default: 8400). */
callbackPort?: number;
/** Protect AuthorizationCode with S256 PKCE (default: true). */
usePkce?: boolean;
/** Client ID for user-assigned managed identity (AzureManagedIdentity). */
managedIdentityClientId?: string;
/**
* Seconds before expiry to trigger proactive refresh (default: 300).
* Keep this well below the token TTL; if it is greater than or equal to
* the TTL, each request refreshes the token.
*/
refreshBufferSecs?: number;
/**
* Opt in to the persistent token cache so short-lived processes reuse one
* session. Only refresh tokens are persisted. Only supported by
* AuthorizationCode and DeviceCode; Azure managed identity is rejected.
* Default: unset (memory only).
*/
tokenCache?: TokenCacheOptions;
}
/**
* Safe, non-secret view of a cached OAuth session, returned by
* {@link OAuthSession.status} and {@link OAuthSession.login}.
*/
export interface SessionStatus {
/**
* Whether a cached session exists that can obtain tokens without
* interactive authentication. Because access tokens are not persisted,
* this is `true` exactly when a refresh token is cached; the next
* connection refreshes with it rather than opening a browser or device
* prompt.
*/
refreshable: boolean;
/** Canonical issuer URL of the cached session. */
issuerUrl: string;
/** Client ID of the cached session. */
clientId: string;
/** Canonical (sorted, de-duplicated) scope set of the cached session. */
scopes: string[];
/** Resource indicator used to obtain the cached session, if configured. */
resource?: string;
/** Provider-specific audience used to obtain the cached session, if configured. */
audience?: string;
/** Flow that produced the cached session. */
flow: string;
/** When the cached session was obtained, as Unix seconds. */
obtainedAt?: number;
}
/** Result of {@link OAuthSession.logout}. */
export interface SessionLogout {
/**
* Whether a cached credential was removed. `false` means no matching
* session was cached; logout is idempotent.
*/
removed: boolean;
}
/**
* Explicit OAuth session lifecycle for the persistent token cache: eager
* `login`, non-secret `status`, and local `logout`.
*
* A session is built from the same {@link OAuthConfig} used to connect
* (including its `tokenCache` options). A connection created with the same
* configuration shares the cache, so logging in here prepares tokens for
* later processes without any database request.
*
* `login` always runs the configured interactive flow and replaces the cached
* session (the most recent login wins). `logout` removes only the local
* credential; it does not revoke anything with the provider and does not sign
* out of a browser SSO session.
*
* @example
* ```typescript
* const config: OAuthConfig = {
* issuerUrl: "https://issuer.example.com",
* clientId: "my-app",
* scopes: ["openid", "offline_access"],
* flow: OAuthFlowType.DeviceCode,
* tokenCache: { cacheDir: "/tmp/my-app/oauth-cache" },
* };
* const session = new OAuthSession(config);
* const status = await session.login();
* ```
*/
export class OAuthSession {
private readonly inner: NativeOAuthSession;
/** Create a session manager for the given OAuth configuration. */
constructor(config: OAuthConfig) {
this.inner = new NativeOAuthSession(config as unknown as NativeOAuthConfig);
}
/**
* Eagerly run the configured authentication flow and store the session.
*
* A successful login always replaces any prior cached session for this
* identity; if the provider does not issue a refresh token (for example
* without `offline_access`), the previous record is removed and the status
* reports `refreshable == false`.
*/
async login(): Promise<SessionStatus> {
return this.inner.login();
}
/**
* Report whether a matching cached session exists, with safe metadata.
*
* This never contacts the identity provider and never exposes token values.
*/
async status(): Promise<SessionStatus> {
return this.inner.status();
}
/**
* Remove the matching local cached credential.
*
* This only deletes the local cache entry. It does not revoke the refresh
* token with the provider and does not sign out of a browser SSO session.
* Repeated calls succeed; `removed` reports whether a credential existed.
*/
async logout(): Promise<SessionLogout> {
return this.inner.logout();
}
}
nodejs/lancedb/otel.ts
+137
View File
@@ -0,0 +1,137 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import {
type Attributes,
type MeterProvider,
type ObservableResult,
metrics,
} from "@opentelemetry/api";
import {
lancedbMetricsCatalog,
registerLancedbMetricsRecorder,
snapshotLancedbMetrics,
} from "./native";
let instrumented = false;
/**
* Register LanceDB metrics as OpenTelemetry observable instruments.
*
* Installs a process-global metrics recorder and creates one observable
* instrument per LanceDB metric (currently object store request counts, bytes,
* latency, errors, and throttles) on the given (or global) `MeterProvider`. The
* configured `MetricReader` then collects them on its own schedule.
*
* Counters and gauges map directly to observable counters/gauges. Because
* OpenTelemetry has no asynchronous histogram instrument, each histogram is
* exported Prometheus-style as cumulative `le` bucket counts (`<name>_bucket`,
* with an `le` attribute) plus `<name>_count` and `<name>_sum`.
*
* Requires `@opentelemetry/api` (a dependency) and, to actually export, an
* OpenTelemetry SDK such as `@opentelemetry/sdk-metrics`.
*
* @param meterProvider The provider to register instruments on. Defaults to the
* global provider from `@opentelemetry/api`.
* @returns `true` if the recorder is installed and instruments are registered.
* `false` if a different `metrics` recorder is already installed in this
* process (only one global recorder is permitted), in which case a warning is
* emitted and no instruments are created. Calling this more than once is safe;
* instruments are created only on the first successful call.
*/
export function instrumentLanceDbMetrics(
meterProvider?: MeterProvider,
): boolean {
if (!registerLancedbMetricsRecorder()) {
console.warn(
"Could not install the LanceDB metrics recorder: another `metrics` " +
"recorder is already installed in this process. LanceDB metrics will " +
"not be exported via OpenTelemetry.",
);
return false;
}
if (instrumented) {
return true;
}
const provider = meterProvider ?? metrics.getMeterProvider();
const meter = provider.getMeter("lancedb");
const scalarCallback = (metricName: string) => (result: ObservableResult) => {
for (const point of snapshotLancedbMetrics()) {
if (point.name === metricName && point.value != null) {
result.observe(point.value, point.attributes);
}
}
};
const bucketCallback = (metricName: string) => (result: ObservableResult) => {
for (const point of snapshotLancedbMetrics()) {
if (point.name !== metricName || point.buckets == null) {
continue;
}
for (const bucket of point.buckets) {
const attributes: Attributes = {
...point.attributes,
le: bucket.le,
};
result.observe(bucket.cumulativeCount, attributes);
}
}
};
const fieldCallback =
(metricName: string, field: "count" | "sum") =>
(result: ObservableResult) => {
for (const point of snapshotLancedbMetrics()) {
if (point.name !== metricName) {
continue;
}
const value = point[field];
if (value != null) {
result.observe(value, point.attributes);
}
}
};
for (const desc of lancedbMetricsCatalog()) {
const unit = desc.unit ?? "";
if (desc.kind === "counter") {
const counter = meter.createObservableCounter(desc.name, {
unit,
description: desc.description,
});
counter.addCallback(scalarCallback(desc.name));
} else if (desc.kind === "gauge") {
const gauge = meter.createObservableGauge(desc.name, {
unit,
description: desc.description,
});
gauge.addCallback(scalarCallback(desc.name));
} else if (desc.kind === "histogram") {
// `_bucket` and `_count` observe cumulative sample counts, not the
// histogram's measured quantity, so they are unitless; only `_sum`
// carries the histogram's unit.
const bucket = meter.createObservableCounter(`${desc.name}_bucket`, {
description: `${desc.description} (cumulative buckets)`,
});
bucket.addCallback(bucketCallback(desc.name));
const count = meter.createObservableCounter(`${desc.name}_count`, {
description: `${desc.description} (count)`,
});
count.addCallback(fieldCallback(desc.name, "count"));
const sum = meter.createObservableCounter(`${desc.name}_sum`, {
unit,
description: `${desc.description} (sum)`,
});
sum.addCallback(fieldCallback(desc.name, "sum"));
}
}
instrumented = true;
return true;
}
nodejs/lancedb/permutation.ts
+202
View File
@@ -0,0 +1,202 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { Connection, LocalConnection } from "./connection.js";
import {
PermutationBuilder as NativePermutationBuilder,
Table as NativeTable,
ShuffleOptions,
SplitCalculatedOptions,
SplitHashOptions,
SplitRandomOptions,
SplitSequentialOptions,
permutationBuilder as nativePermutationBuilder,
} from "./native.js";
import { LocalTable, Table } from "./table";
/**
* A PermutationBuilder for creating data permutations with splits, shuffling, and filtering.
*
* This class provides a TypeScript wrapper around the native Rust PermutationBuilder,
* offering methods to configure data splits, shuffling, and filtering before executing
* the permutation to create a new table.
*/
export class PermutationBuilder {
private inner: NativePermutationBuilder;
/**
* @hidden
*/
constructor(inner: NativePermutationBuilder) {
this.inner = inner;
}
/**
* Configure the permutation to be persisted.
*
* @param connection - The connection to persist the permutation to
* @param tableName - The name of the table to create
* @returns A new PermutationBuilder instance
* @example
* ```ts
* builder.persist(connection, "permutation_table");
* ```
*/
persist(connection: Connection, tableName: string): PermutationBuilder {
const localConnection = connection as LocalConnection;
const newInner = this.inner.persist(localConnection.inner, tableName);
return new PermutationBuilder(newInner);
}
/**
* Configure random splits for the permutation.
*
* @param options - Configuration for random splitting
* @returns A new PermutationBuilder instance
* @example
* ```ts
* // Split by ratios
* builder.splitRandom({ ratios: [0.7, 0.3], seed: 42 });
*
* // Split by counts
* builder.splitRandom({ counts: [1000, 500], seed: 42 });
*
* // Split with fixed size
* builder.splitRandom({ fixed: 100, seed: 42 });
* ```
*/
splitRandom(options: SplitRandomOptions): PermutationBuilder {
const newInner = this.inner.splitRandom(options);
return new PermutationBuilder(newInner);
}
/**
* Configure hash-based splits for the permutation.
*
* @param options - Configuration for hash-based splitting
* @returns A new PermutationBuilder instance
* @example
* ```ts
* builder.splitHash({
* columns: ["user_id"],
* splitWeights: [70, 30],
* discardWeight: 0
* });
* ```
*/
splitHash(options: SplitHashOptions): PermutationBuilder {
const newInner = this.inner.splitHash(options);
return new PermutationBuilder(newInner);
}
/**
* Configure sequential splits for the permutation.
*
* @param options - Configuration for sequential splitting
* @returns A new PermutationBuilder instance
* @example
* ```ts
* // Split by ratios
* builder.splitSequential({ ratios: [0.8, 0.2] });
*
* // Split by counts
* builder.splitSequential({ counts: [800, 200] });
*
* // Split with fixed size
* builder.splitSequential({ fixed: 1000 });
* ```
*/
splitSequential(options: SplitSequentialOptions): PermutationBuilder {
const newInner = this.inner.splitSequential(options);
return new PermutationBuilder(newInner);
}
/**
* Configure calculated splits for the permutation.
*
* @param options - Configuration for calculated splitting
* @returns A new PermutationBuilder instance
* @example
* ```ts
* builder.splitCalculated({ calculation: "user_id % 3" });
* ```
*/
splitCalculated(options: SplitCalculatedOptions): PermutationBuilder {
const newInner = this.inner.splitCalculated(options);
return new PermutationBuilder(newInner);
}
/**
* Configure shuffling for the permutation.
*
* @param options - Configuration for shuffling
* @returns A new PermutationBuilder instance
* @example
* ```ts
* // Basic shuffle
* builder.shuffle({ seed: 42 });
*
* // Shuffle with clump size
* builder.shuffle({ seed: 42, clumpSize: 10 });
* ```
*/
shuffle(options: ShuffleOptions): PermutationBuilder {
const newInner = this.inner.shuffle(options);
return new PermutationBuilder(newInner);
}
/**
* Configure filtering for the permutation.
*
* @param filter - SQL filter expression
* @returns A new PermutationBuilder instance
* @example
* ```ts
* builder.filter("age > 18 AND status = 'active'");
* ```
*/
filter(filter: string): PermutationBuilder {
const newInner = this.inner.filter(filter);
return new PermutationBuilder(newInner);
}
/**
* Execute the permutation and create the destination table.
*
* @returns A Promise that resolves to the new Table instance
* @example
* ```ts
* const permutationTable = await builder.execute();
* console.log(`Created table: ${permutationTable.name}`);
* ```
*/
async execute(): Promise<Table> {
const nativeTable: NativeTable = await this.inner.execute();
return new LocalTable(nativeTable);
}
}
/**
* Create a permutation builder for the given table.
*
* @param table - The source table to create a permutation from
* @returns A PermutationBuilder instance
* @example
* ```ts
* const builder = permutationBuilder(sourceTable, "training_data")
* .splitRandom({ ratios: [0.8, 0.2], seed: 42 })
* .shuffle({ seed: 123 });
*
* const trainingTable = await builder.execute();
* ```
*/
export function permutationBuilder(table: Table): PermutationBuilder {
// Extract the inner native table from the TypeScript wrapper
const localTable = table as LocalTable;
// Access inner through type assertion since it's private
const nativeBuilder = nativePermutationBuilder(
// biome-ignore lint/suspicious/noExplicitAny: need access to private variable
(localTable as any).inner,
);
return new PermutationBuilder(nativeBuilder);
}
nodejs/lancedb/query.ts
+583 -163
View File
File diff suppressed because it is too large Load Diff
nodejs/lancedb/sanitize.ts
+370 -43
View File
@@ -3,13 +3,14 @@
// The utilities in this file help sanitize data from the user's arrow
// library into the types expected by vectordb's arrow library. Node
// generally allows for mulitple versions of the same library (and sometimes
// generally allows for multiple versions of the same library (and sometimes
// even multiple copies of the same version) to be installed at the same
// time. However, arrow-js uses instanceof which expected that the input
// comes from the exact same library instance. This is not always the case
// and so we must sanitize the input to ensure that it is compatible.
import { BufferType, Data } from "apache-arrow";
import * as arrowModule from "apache-arrow";
import { BufferType, Data, Vector } from "apache-arrow";
import type { IntBitWidth, TKeys, TimeBitWidth } from "apache-arrow/type";
import {
Binary,
@@ -43,6 +44,8 @@ import {
Interval,
IntervalDayTime,
IntervalYearMonth,
LargeBinary,
LargeUtf8,
List,
Map_,
Null,
@@ -74,23 +77,104 @@ import {
Utf8,
} from "./arrow";
/**
* Whether `value` was created by *this* copy of apache-arrow's `ctor`.
*
* Arrow 21 overrides `Symbol.hasInstance` on `Schema`, `Field`, `Table`,
* `RecordBatch`, `Vector` and `Data` so that `instanceof` also matches objects
* built by a different copy of the library. Those are exactly the objects
* this module has to rebuild: the `DataType` instances nested inside them are
* deliberately *not* given that treatment upstream, so Arrow's own builders and
* writers reject them. Walk the real prototype chain instead of trusting
* `instanceof`.
*/
function isOwnInstance<T>(
value: unknown,
// biome-ignore lint/suspicious/noExplicitAny: matches any class constructor
ctor: abstract new (...args: any[]) => T,
): value is T {
return (
typeof value === "object" &&
value !== null &&
Object.prototype.isPrototypeOf.call(ctor.prototype, value)
);
}
/**
* Types added to Arrow JS after release 18, keyed by the `Type` enum value
* they use. The library compiles and runs against every Arrow release in the
* peer range, so these are looked up by name at runtime instead of imported:
* an older Arrow simply does not have the class.
*/
type NewerTypeName =
| "IntervalMonthDayNano"
| "LargeList"
| "BinaryView"
| "Utf8View";
const NEWER_TYPES: ReadonlyArray<{
name: NewerTypeName;
typeId: number;
since: number;
}> = [
{ name: "IntervalMonthDayNano", typeId: -31, since: 20 },
{ name: "LargeList", typeId: 21, since: 21 },
{ name: "BinaryView", typeId: 23, since: 21 },
{ name: "Utf8View", typeId: 24, since: 21 },
];
// biome-ignore lint/suspicious/noExplicitAny: constructor arity varies by type
type DataTypeConstructor = new (...args: any[]) => DataType;
function newerTypeConstructor(name: NewerTypeName): DataTypeConstructor {
const ctor = (arrowModule as unknown as Record<string, unknown>)[name];
if (typeof ctor !== "function") {
const since = NEWER_TYPES.find((t) => t.name === name)?.since;
throw new Error(
`The Arrow type ${name} requires apache-arrow ${since} or newer, ` +
"but the installed apache-arrow does not provide it",
);
}
return ctor as DataTypeConstructor;
}
type SanitizationContext = {
types: WeakMap<object, DataType>;
vectors: WeakMap<object, Vector>;
data: WeakMap<object, Data<DataType>>;
};
function createSanitizationContext(): SanitizationContext {
return {
types: new WeakMap(),
vectors: new WeakMap(),
data: new WeakMap(),
};
}
export function sanitizeMetadata(
metadataLike?: unknown,
): Map<string, string> | undefined {
if (metadataLike === undefined || metadataLike === null) {
return undefined;
}
if (!(metadataLike instanceof Map)) {
let entries: IterableIterator<[unknown, unknown]>;
try {
entries = Map.prototype.entries.call(metadataLike);
} catch {
throw Error("Expected metadata, if present, to be a Map<string, string>");
}
for (const item of metadataLike) {
if (!(typeof item[0] === "string" || !(typeof item[1] === "string"))) {
const metadata = new Map<string, string>();
for (const [key, value] of entries) {
if (typeof key !== "string" || typeof value !== "string") {
throw Error(
"Expected metadata, if present, to be a Map<string, string> but it had non-string keys or values",
);
}
metadata.set(key, value);
}
return metadataLike as Map<string, string>;
return metadata;
}
export function sanitizeInt(typeLike: object) {
@@ -186,27 +270,68 @@ export function sanitizeInterval(typeLike: object) {
}
export function sanitizeList(typeLike: object) {
return sanitizeListWithContext(typeLike, createSanitizationContext());
}
function sanitizeListChildWithContext(
typeLike: object,
context: SanitizationContext,
typeName: string,
): Field {
if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
throw Error(
"Expected a List type to have an array-like `children` property",
`Expected a ${typeName} type to have an array-like \`children\` property`,
);
}
if (typeLike.children.length !== 1) {
throw Error("Expected a List type to have exactly one child");
throw Error(`Expected a ${typeName} type to have exactly one child`);
}
return new List(sanitizeField(typeLike.children[0]));
return sanitizeFieldWithContext(typeLike.children[0], context);
}
function sanitizeListWithContext(
typeLike: object,
context: SanitizationContext,
) {
return new List(sanitizeListChildWithContext(typeLike, context, "List"));
}
function sanitizeLargeListWithContext(
typeLike: object,
context: SanitizationContext,
) {
const LargeList = newerTypeConstructor("LargeList");
return new LargeList(
sanitizeListChildWithContext(typeLike, context, "LargeList"),
);
}
export function sanitizeStruct(typeLike: object) {
return sanitizeStructWithContext(typeLike, createSanitizationContext());
}
function sanitizeStructWithContext(
typeLike: object,
context: SanitizationContext,
) {
if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
throw Error(
"Expected a Struct type to have an array-like `children` property",
);
}
return new Struct(typeLike.children.map((child) => sanitizeField(child)));
return new Struct(
typeLike.children.map((child) => sanitizeFieldWithContext(child, context)),
);
}
export function sanitizeUnion(typeLike: object) {
return sanitizeUnionWithContext(typeLike, createSanitizationContext());
}
function sanitizeUnionWithContext(
typeLike: object,
context: SanitizationContext,
) {
if (
!("typeIds" in typeLike) ||
!("mode" in typeLike) ||
@@ -226,7 +351,7 @@ export function sanitizeUnion(typeLike: object) {
typeLike.mode,
// biome-ignore lint/suspicious/noExplicitAny: skip
typeLike.typeIds as any,
typeLike.children.map((child) => sanitizeField(child)),
typeLike.children.map((child) => sanitizeFieldWithContext(child, context)),
);
}
@@ -234,6 +359,19 @@ export function sanitizeTypedUnion(
typeLike: object,
// eslint-disable-next-line @typescript-eslint/naming-convention
UnionType: typeof DenseUnion | typeof SparseUnion,
) {
return sanitizeTypedUnionWithContext(
typeLike,
UnionType,
createSanitizationContext(),
);
}
function sanitizeTypedUnionWithContext(
typeLike: object,
// eslint-disable-next-line @typescript-eslint/naming-convention
UnionType: typeof DenseUnion | typeof SparseUnion,
context: SanitizationContext,
) {
if (!("typeIds" in typeLike)) {
throw Error(
@@ -248,7 +386,7 @@ export function sanitizeTypedUnion(
return new UnionType(
typeLike.typeIds as Int32Array | number[],
typeLike.children.map((child) => sanitizeField(child)),
typeLike.children.map((child) => sanitizeFieldWithContext(child, context)),
);
}
@@ -262,6 +400,16 @@ export function sanitizeFixedSizeBinary(typeLike: object) {
}
export function sanitizeFixedSizeList(typeLike: object) {
return sanitizeFixedSizeListWithContext(
typeLike,
createSanitizationContext(),
);
}
function sanitizeFixedSizeListWithContext(
typeLike: object,
context: SanitizationContext,
) {
if (!("listSize" in typeLike) || typeof typeLike.listSize !== "number") {
throw Error("Expected a FixedSizeList type to have a `listSize` property");
}
@@ -275,11 +423,18 @@ export function sanitizeFixedSizeList(typeLike: object) {
}
return new FixedSizeList(
typeLike.listSize,
sanitizeField(typeLike.children[0]),
sanitizeFieldWithContext(typeLike.children[0], context),
);
}
export function sanitizeMap(typeLike: object) {
return sanitizeMapWithContext(typeLike, createSanitizationContext());
}
function sanitizeMapWithContext(
typeLike: object,
context: SanitizationContext,
) {
if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
throw Error(
"Expected a Map type to have an array-like `children` property",
@@ -288,10 +443,12 @@ export function sanitizeMap(typeLike: object) {
if (!("keysSorted" in typeLike) || typeof typeLike.keysSorted !== "boolean") {
throw Error("Expected a Map type to have a `keysSorted` property");
}
if (typeLike.children.length !== 1) {
throw Error("Expected a Map type to have exactly one child");
}
return new Map_(
// biome-ignore lint/suspicious/noExplicitAny: skip
typeLike.children.map((field) => sanitizeField(field)) as any,
sanitizeFieldWithContext(typeLike.children[0], context),
typeLike.keysSorted,
);
}
@@ -304,6 +461,13 @@ export function sanitizeDuration(typeLike: object) {
}
export function sanitizeDictionary(typeLike: object) {
return sanitizeDictionaryWithContext(typeLike, createSanitizationContext());
}
function sanitizeDictionaryWithContext(
typeLike: object,
context: SanitizationContext,
) {
if (!("id" in typeLike) || typeof typeLike.id !== "number") {
throw Error("Expected a Dictionary type to have an `id` property");
}
@@ -317,8 +481,8 @@ export function sanitizeDictionary(typeLike: object) {
throw Error("Expected a Dictionary type to have an `isOrdered` property");
}
return new Dictionary(
sanitizeType(typeLike.dictionary),
sanitizeType(typeLike.indices) as TKeys,
sanitizeTypeWithContext(typeLike.dictionary, context),
sanitizeTypeWithContext(typeLike.indices, context) as TKeys,
typeLike.id,
typeLike.isOrdered,
);
@@ -326,9 +490,23 @@ export function sanitizeDictionary(typeLike: object) {
// biome-ignore lint/suspicious/noExplicitAny: skip
export function sanitizeType(typeLike: unknown): DataType<any> {
return sanitizeTypeWithContext(typeLike, createSanitizationContext());
}
function sanitizeTypeWithContext(
typeLike: unknown,
context: SanitizationContext,
): DataType {
if (typeof typeLike === "string") {
return dataTypeFromName(typeLike);
}
if (typeof typeLike !== "object" || typeLike === null) {
throw Error("Expected a Type but object was null/undefined");
}
const cached = context.types.get(typeLike);
if (cached !== undefined) {
return cached;
}
if (
!("typeId" in typeLike) ||
!(
@@ -347,6 +525,35 @@ export function sanitizeType(typeLike: unknown): DataType<any> {
throw Error("Type's typeId property was not a function or number");
}
const type = sanitizeTypeById(typeLike, typeId, context);
context.types.set(typeLike, type);
return type;
}
function sanitizeNewerTypeById(
typeLike: object,
typeId: number,
context: SanitizationContext,
): DataType | undefined {
const entry = NEWER_TYPES.find((t) => t.typeId === typeId);
if (entry === undefined) {
return undefined;
}
if (entry.name === "LargeList") {
return sanitizeLargeListWithContext(typeLike, context);
}
return new (newerTypeConstructor(entry.name))();
}
function sanitizeTypeById(
typeLike: object,
typeId: Type,
context: SanitizationContext,
): DataType {
const newer = sanitizeNewerTypeById(typeLike, typeId, context);
if (newer !== undefined) {
return newer;
}
switch (typeId) {
case Type.NONE:
throw Error("Received a Type with a typeId of NONE");
@@ -358,8 +565,12 @@ export function sanitizeType(typeLike: unknown): DataType<any> {
return sanitizeFloat(typeLike);
case Type.Binary:
return new Binary();
case Type.LargeBinary:
return new LargeBinary();
case Type.Utf8:
return new Utf8();
case Type.LargeUtf8:
return new LargeUtf8();
case Type.Bool:
return new Bool();
case Type.Decimal:
@@ -373,21 +584,21 @@ export function sanitizeType(typeLike: unknown): DataType<any> {
case Type.Interval:
return sanitizeInterval(typeLike);
case Type.List:
return sanitizeList(typeLike);
return sanitizeListWithContext(typeLike, context);
case Type.Struct:
return sanitizeStruct(typeLike);
return sanitizeStructWithContext(typeLike, context);
case Type.Union:
return sanitizeUnion(typeLike);
return sanitizeUnionWithContext(typeLike, context);
case Type.FixedSizeBinary:
return sanitizeFixedSizeBinary(typeLike);
case Type.FixedSizeList:
return sanitizeFixedSizeList(typeLike);
return sanitizeFixedSizeListWithContext(typeLike, context);
case Type.Map:
return sanitizeMap(typeLike);
return sanitizeMapWithContext(typeLike, context);
case Type.Duration:
return sanitizeDuration(typeLike);
case Type.Dictionary:
return sanitizeDictionary(typeLike);
return sanitizeDictionaryWithContext(typeLike, context);
case Type.Int8:
return new Int8();
case Type.Int16:
@@ -431,9 +642,9 @@ export function sanitizeType(typeLike: unknown): DataType<any> {
case Type.TimestampSecond:
return sanitizeTypedTimestamp(typeLike, TimestampSecond);
case Type.DenseUnion:
return sanitizeTypedUnion(typeLike, DenseUnion);
return sanitizeTypedUnionWithContext(typeLike, DenseUnion, context);
case Type.SparseUnion:
return sanitizeTypedUnion(typeLike, SparseUnion);
return sanitizeTypedUnionWithContext(typeLike, SparseUnion, context);
case Type.IntervalDayTime:
return new IntervalDayTime();
case Type.IntervalYearMonth:
@@ -447,12 +658,19 @@ export function sanitizeType(typeLike: unknown): DataType<any> {
case Type.DurationSecond:
return new DurationSecond();
default:
throw new Error("Unrecoginized type id in schema: " + typeId);
throw new Error("Unrecognized type id in schema: " + typeId);
}
}
export function sanitizeField(fieldLike: unknown): Field {
if (fieldLike instanceof Field) {
return sanitizeFieldWithContext(fieldLike, createSanitizationContext());
}
function sanitizeFieldWithContext(
fieldLike: unknown,
context: SanitizationContext,
): Field {
if (isOwnInstance(fieldLike, Field)) {
return fieldLike;
}
if (typeof fieldLike !== "object" || fieldLike === null) {
@@ -467,7 +685,15 @@ export function sanitizeField(fieldLike: unknown): Field {
"The field passed in is missing a `type`/`name`/`nullable` property",
);
}
const type = sanitizeType(fieldLike.type);
let type: DataType;
try {
type = sanitizeTypeWithContext(fieldLike.type, context);
} catch (error: unknown) {
throw Error(
`Unable to sanitize type for field: ${fieldLike.name} due to error: ${error}`,
{ cause: error },
);
}
const name = fieldLike.name;
if (!(typeof name === "string")) {
throw Error("The field passed in had a non-string `name` property");
@@ -491,7 +717,14 @@ export function sanitizeField(fieldLike: unknown): Field {
* than lancedb is using.
*/
export function sanitizeSchema(schemaLike: SchemaLike): Schema {
if (schemaLike instanceof Schema) {
return sanitizeSchemaWithContext(schemaLike, createSanitizationContext());
}
function sanitizeSchemaWithContext(
schemaLike: SchemaLike,
context: SanitizationContext,
): Schema {
if (isOwnInstance(schemaLike, Schema)) {
return schemaLike;
}
if (typeof schemaLike !== "object" || schemaLike === null) {
@@ -512,13 +745,13 @@ export function sanitizeSchema(schemaLike: SchemaLike): Schema {
);
}
const sanitizedFields = schemaLike.fields.map((field) =>
sanitizeField(field),
sanitizeFieldWithContext(field, context),
);
return new Schema(sanitizedFields, metadata);
}
export function sanitizeTable(tableLike: TableLike): Table {
if (tableLike instanceof Table) {
if (isOwnInstance(tableLike, Table)) {
return tableLike;
}
if (typeof tableLike !== "object" || tableLike === null) {
@@ -534,14 +767,19 @@ export function sanitizeTable(tableLike: TableLike): Table {
"The table passed in does not appear to be a table (no 'columns' property)",
);
}
const schema = sanitizeSchema(tableLike.schema);
const batches = tableLike.batches.map(sanitizeRecordBatch);
const context = createSanitizationContext();
const schema = sanitizeSchemaWithContext(tableLike.schema, context);
const batches = tableLike.batches.map((batch) =>
sanitizeRecordBatch(batch, context),
);
return new Table(schema, batches);
}
function sanitizeRecordBatch(batchLike: RecordBatchLike): RecordBatch {
if (batchLike instanceof RecordBatch) {
function sanitizeRecordBatch(
batchLike: RecordBatchLike,
context: SanitizationContext,
): RecordBatch {
if (isOwnInstance(batchLike, RecordBatch)) {
return batchLike;
}
if (typeof batchLike !== "object" || batchLike === null) {
@@ -557,19 +795,54 @@ function sanitizeRecordBatch(batchLike: RecordBatchLike): RecordBatch {
"The record batch passed in does not appear to be a record batch (no 'data' property)",
);
}
const schema = sanitizeSchema(batchLike.schema);
const data = sanitizeData(batchLike.data);
const schema = sanitizeSchemaWithContext(batchLike.schema, context);
const data = sanitizeData(batchLike.data, context) as Data<Struct>;
return new RecordBatch(schema, data);
}
type DictionaryVectorLike = {
data: readonly DataLike[];
};
type DictionaryDataLike = DataLike & {
dictionary?: DictionaryVectorLike;
};
// Arrow 21 added `variadicBuffers` to `Data` for the Utf8View / BinaryView
// types. Older Arrow versions do not have the property, so it is optional.
type VariadicDataLike = DataLike & {
variadicBuffers?: ReadonlyArray<Uint8Array>;
};
function sanitizeData(
dataLike: DataLike,
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
): import("apache-arrow").Data<Struct<any>> {
if (dataLike instanceof Data) {
context: SanitizationContext,
): Data<DataType> {
if (isOwnInstance(dataLike, Data)) {
return dataLike;
}
return new Data(
dataLike.type,
const cachedData = context.data.get(dataLike);
if (cachedData !== undefined) {
return cachedData;
}
const dictionaryLike = (dataLike as DictionaryDataLike).dictionary;
let dictionary: Vector | undefined;
if (dictionaryLike !== undefined) {
dictionary = context.vectors.get(dictionaryLike);
if (dictionary === undefined) {
dictionary = new Vector(
dictionaryLike.data.map((data) => sanitizeData(data, context)),
);
context.vectors.set(dictionaryLike, dictionary);
}
}
// Arrow 21 added an eighth constructor parameter (`variadicBuffers`); older
// releases ignore the extra argument but their typings do not declare it.
const DataCtor = Data as unknown as new (
...args: unknown[]
) => Data<DataType>;
const data = new DataCtor(
sanitizeTypeWithContext(dataLike.type, context),
dataLike.offset,
dataLike.length,
dataLike.nullCount,
@@ -579,5 +852,59 @@ function sanitizeData(
[BufferType.VALIDITY]: dataLike.nullBitmap,
[BufferType.TYPE]: dataLike.typeIds,
},
dataLike.children.map((child) => sanitizeData(child, context)),
dictionary,
(dataLike as VariadicDataLike).variadicBuffers,
);
context.data.set(dataLike, data);
return data;
}
const constructorsByTypeName = {
null: () => new Null(),
binary: () => new Binary(),
largebinary: () => new LargeBinary(),
binaryview: () => new (newerTypeConstructor("BinaryView"))(),
utf8: () => new Utf8(),
largeutf8: () => new LargeUtf8(),
utf8view: () => new (newerTypeConstructor("Utf8View"))(),
bool: () => new Bool(),
int8: () => new Int8(),
int16: () => new Int16(),
int32: () => new Int32(),
int64: () => new Int64(),
uint8: () => new Uint8(),
uint16: () => new Uint16(),
uint32: () => new Uint32(),
uint64: () => new Uint64(),
float16: () => new Float16(),
float32: () => new Float32(),
float64: () => new Float64(),
datemillisecond: () => new DateMillisecond(),
dateday: () => new DateDay(),
timenanosecond: () => new TimeNanosecond(),
timemicrosecond: () => new TimeMicrosecond(),
timemillisecond: () => new TimeMillisecond(),
timesecond: () => new TimeSecond(),
intervaldaytime: () => new IntervalDayTime(),
intervalyearmonth: () => new IntervalYearMonth(),
intervalmonthdaynano: () =>
new (newerTypeConstructor("IntervalMonthDayNano"))(),
durationnanosecond: () => new DurationNanosecond(),
durationmicrosecond: () => new DurationMicrosecond(),
durationmillisecond: () => new DurationMillisecond(),
durationsecond: () => new DurationSecond(),
} as const;
type MappableTypeName = keyof typeof constructorsByTypeName;
export function dataTypeFromName(typeName: string): DataType {
const normalizedTypeName = typeName.toLowerCase() as MappableTypeName;
const _constructor = constructorsByTypeName[normalizedTypeName];
if (!_constructor) {
throw new Error("Unrecognized type name in schema: " + typeName);
}
return _constructor();
}
nodejs/lancedb/scannable.ts
+274
View File
@@ -0,0 +1,274 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import {
Table as ArrowTable,
RecordBatch,
RecordBatchReader,
Schema,
} from "apache-arrow";
import {
fromRecordBatchToStreamBuffer,
fromTableToBuffer,
makeEmptyTable,
} from "./arrow";
import { NapiScannable } from "./native.js";
export interface ScannableOptions {
/** Hint about the number of rows. Not validated against the stream. */
numRows?: number;
/**
* Whether the source can be scanned more than once. Defaults to `true` for
* `fromTable` / `fromFactory` and `false` for `fromIterable` /
* `fromRecordBatchReader`.
*/
rescannable?: boolean;
}
/**
* A data source that can be scanned as a stream of Arrow `RecordBatch`es.
*
* `Scannable` wraps the schema + optional row count + rescannable flag and
* a callback that yields batches one at a time. It is passed to consumers
* (e.g. `Table.add`, `createTable`, `mergeInsert` — follow-up work) that
* need to pull data without materializing the full dataset in JS memory.
*
* Batches cross the JS↔Rust boundary as Arrow IPC Stream messages; a fresh
* writer serializes each batch, and the Rust side decodes it with
* `arrow_ipc::reader::StreamReader`. One batch is in flight at a time.
*/
export class Scannable {
readonly schema: Schema;
readonly numRows: number | null;
readonly rescannable: boolean;
/** @hidden */
private readonly native: NapiScannable;
private constructor(
native: NapiScannable,
schema: Schema,
numRows: number | null,
rescannable: boolean,
) {
this.native = native;
this.schema = schema;
this.numRows = numRows;
this.rescannable = rescannable;
}
/** @hidden Access the native handle for passing through to Rust consumers. */
get inner(): NapiScannable {
return this.native;
}
/**
* Build a Scannable from an explicit schema and a factory that returns a
* fresh batch iterator on each call.
*
* The factory is invoked once per scan. Each iterator yields
* `RecordBatch`es matching the declared schema. Use this when you need
* direct control over the pull loop — for example, to wrap a streaming
* source whose batches are produced lazily.
*
* @param schema - The Arrow schema of the produced batches.
* @param factory - Called at the start of each scan to produce a batch
* iterator. Must be idempotent when `rescannable` is true.
* @param opts - Optional hints. `rescannable` defaults to `true`; set to
* `false` if calling `factory()` twice would not reproduce the same data.
*/
static async fromFactory(
schema: Schema,
factory: () =>
| AsyncIterable<RecordBatch>
| Iterable<RecordBatch>
| AsyncIterator<RecordBatch>
| Iterator<RecordBatch>,
opts: ScannableOptions = {},
): Promise<Scannable> {
const numRows = opts.numRows ?? null;
if (numRows != null && !Number.isInteger(numRows)) {
throw new TypeError("numRows must be an integer");
}
const rescannable = opts.rescannable ?? true;
let iter: AsyncIterator<RecordBatch> | Iterator<RecordBatch> | null = null;
const getNextBatch = async (isStart: boolean): Promise<Buffer | null> => {
// `isStart` is true on the first pull of every new scan_as_stream.
// Drop any cached iterator so factory() is re-invoked for the next scan
if (isStart) {
iter = null;
}
if (iter === null) {
iter = normalizeIterator(factory());
}
const result = await iter.next();
if (result.done) {
iter = null;
return null;
}
return fromRecordBatchToStreamBuffer(result.value);
};
const schemaBuf = await fromTableToBuffer(makeEmptyTable(schema));
const native = new NapiScannable(
schemaBuf,
numRows,
rescannable,
getNextBatch,
);
return new Scannable(native, schema, numRows, rescannable);
}
/**
* Build a Scannable from an in-memory Arrow `Table`. Always rescannable;
* the table's batches are replayed on each scan.
*
* The table's row count is authoritative: `opts.numRows` must either be
* omitted or equal to `table.numRows`. `opts.rescannable` of `false` is
* rejected because in-memory Tables are always rescannable.
*/
static async fromTable(
table: ArrowTable,
opts: ScannableOptions = {},
): Promise<Scannable> {
if (opts.numRows != null && opts.numRows !== table.numRows) {
throw new TypeError(
`opts.numRows (${opts.numRows}) does not match table.numRows (${table.numRows}). ` +
`The table's row count is authoritative; omit numRows or pass the matching value.`,
);
}
if (opts.rescannable === false) {
throw new TypeError(
`fromTable does not accept rescannable: false. ` +
`In-memory Arrow Tables are always rescannable; omit the option or pass true.`,
);
}
return Scannable.fromFactory(table.schema, () => table.batches, {
numRows: table.numRows,
rescannable: true,
});
}
/**
* Build a Scannable from an iterable of `RecordBatch`es. `rescannable`
* defaults to `false`. Pass an explicit schema so the consumer can
* validate before any batch is pulled.
*
* `opts.rescannable: true` is honest for replayable iterables (Arrays,
* Sets, or custom iterables whose `[Symbol.iterator]()` returns a fresh
* iterator each call). It is rejected for one-shot iterables (generators,
* async generators, or already-an-iterator inputs) because their
* `[Symbol.iterator]()` returns the same exhausted object on the second
* scan. For replayable sources outside this shape, use
* `fromFactory(schema, () => createIter(), { rescannable: true })`.
*
* Note: when `opts.rescannable` is `true`, the constructor calls
* `[Symbol.iterator]()` once on the input to perform the structural check.
*/
static async fromIterable(
schema: Schema,
iter: AsyncIterable<RecordBatch> | Iterable<RecordBatch>,
opts: ScannableOptions = {},
): Promise<Scannable> {
if (opts.rescannable === true && isOneShotIterable(iter)) {
throw new TypeError(
`fromIterable: rescannable: true is not honest for one-shot iterables ` +
`(generators, async generators, or iterators where [Symbol.iterator]() ` +
`returns the same object). The source would be exhausted after the first scan. ` +
`Use fromFactory(schema, () => createIter(), { rescannable: true }) for sources ` +
`where each call mints a fresh iterator.`,
);
}
return Scannable.fromFactory(schema, () => iter, {
numRows: opts.numRows,
rescannable: opts.rescannable ?? false,
});
}
/**
* Build a Scannable from an Arrow `RecordBatchReader`. A reader can only
* be consumed once; `rescannable` defaults to `false`.
*
* The reader must already be opened (via `.open()`) so its `.schema` is
* populated. `RecordBatchReader.from(...)` returns an unopened reader.
*
* `opts.rescannable: true` is rejected because `RecordBatchReader` is a
* self-iterator (its `[Symbol.iterator]()` returns itself), and this
* constructor does not call `reader.reset()` between scans, so a second
* scan would always see an exhausted reader. For genuinely replayable
* sources, use
* `fromFactory(schema, () => openReader(), { rescannable: true })`,
* which mints a fresh reader on each scan.
*/
static async fromRecordBatchReader(
reader: RecordBatchReader,
opts: ScannableOptions = {},
): Promise<Scannable> {
if (opts.rescannable === true) {
throw new TypeError(
`fromRecordBatchReader does not accept rescannable: true. ` +
`RecordBatchReader is a self-iterator (its [Symbol.iterator]() ` +
`returns itself) and would be exhausted after the first scan. ` +
`Use fromFactory(schema, () => openReader(), { rescannable: true }) ` +
`for sources where each call mints a fresh reader.`,
);
}
return Scannable.fromFactory(reader.schema, () => reader, {
numRows: opts.numRows,
rescannable: false,
});
}
}
function normalizeIterator<T>(
source: AsyncIterable<T> | Iterable<T> | AsyncIterator<T> | Iterator<T>,
): AsyncIterator<T> | Iterator<T> {
if (source == null) {
throw new TypeError("Scannable factory returned null/undefined");
}
if (
typeof (source as AsyncIterable<T>)[Symbol.asyncIterator] === "function"
) {
return (source as AsyncIterable<T>)[Symbol.asyncIterator]();
}
if (typeof (source as Iterable<T>)[Symbol.iterator] === "function") {
return (source as Iterable<T>)[Symbol.iterator]();
}
// Already an iterator (has `.next`).
if (typeof (source as Iterator<T>).next === "function") {
return source as Iterator<T>;
}
throw new TypeError("Scannable factory returned a non-iterable value");
}
// A "self-iterator" returns the same object from `[Symbol.iterator]()` /
// `[Symbol.asyncIterator]()`. Generators behave this way, so they exhaust
// after one pass. Replayable iterables (Array, Set, custom) return a fresh
// iterator each call. Detection mirrors `normalizeIterator`'s ordering so
// classification matches scan-time behavior.
function isOneShotIterable(
source: AsyncIterable<unknown> | Iterable<unknown>,
): boolean {
// null/undefined are not one-shot in any meaningful sense; let
// `normalizeIterator` raise the actual error at scan time.
if (source == null) return false;
const ref = source as unknown;
if (
typeof (source as AsyncIterable<unknown>)[Symbol.asyncIterator] ===
"function"
) {
const it = (source as AsyncIterable<unknown>)[
Symbol.asyncIterator
]() as unknown;
return it === ref;
}
if (typeof (source as Iterable<unknown>)[Symbol.iterator] === "function") {
const it = (source as Iterable<unknown>)[Symbol.iterator]() as unknown;
return it === ref;
}
// Already-an-iterator (has `.next` but no `Symbol.iterator`) is by
// definition one-shot.
if (typeof (source as { next?: unknown }).next === "function") return true;
return false;
}
nodejs/lancedb/schema.ts
+649
View File
@@ -0,0 +1,649 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import {
Binary,
Bool,
DataType,
Dictionary,
Field,
FixedSizeList,
Float32,
Float64,
Int32,
Int64,
List,
Schema,
Struct,
Utf8,
util as arrowUtil,
} from "apache-arrow";
import { typedArrayToArrowType } from "./arrow_type";
import { sanitizeType } from "./sanitize";
type InferenceOptions = {
dictionaryEncodeStrings: boolean;
vectorColumns: Record<string, { type: unknown }>;
};
/**
* Infer the Arrow schema represented by a set of records.
*
* This is the intentionally small interface to schema inference. The stateful
* details of combining partial type evidence are encapsulated below so callers
* only need to provide records, an optional schema, and inference options.
*/
export function inferSchema(
data: Array<Record<string, unknown>>,
schema: Schema | undefined,
options: InferenceOptions,
): Schema {
return new SchemaInferrer(schema, options).infer(data);
}
class SchemaInferrer {
private readonly fields = new FieldTree();
constructor(
private readonly providedSchema: Schema | undefined,
private readonly options: InferenceOptions,
) {}
infer(data: Array<Record<string, unknown>>): Schema {
for (const [row, record] of data.entries()) {
for (const [path, value] of recordPathsAndValues(record)) {
this.observe(path, value, row);
}
}
return this.providedSchema === undefined
? new Schema(fieldsFromTree(this.fields))
: new Schema(matchingFields(this.providedSchema.fields, this.fields));
}
private observe(path: string[], value: unknown, row: number): void {
assertSupportedValue(value, path.join("."), row);
const current = this.fields.get(path);
if (current === undefined) {
this.addField(path, value, row);
} else if (this.providedSchema === undefined) {
this.updateInferredField(path, value, row, current);
} else if (current instanceof DeferredTypeEvidence) {
if (value != null) {
this.addSchemaField(this.providedSchema, path, value, row);
}
} else if (current instanceof FieldTree || DataType.isStruct(current)) {
assertStructValue(path, value, row);
}
}
private addField(path: string[], value: unknown, row: number): void {
if (this.providedSchema !== undefined) {
this.addSchemaField(this.providedSchema, path, value, row);
return;
}
const evidence =
this.inferType(value, path) ?? DeferredTypeEvidence.from(value, row);
if (evidence === undefined) {
throw typeInferenceError(path, row);
}
const conflict = this.fields.set(
path,
evidence,
(existing) =>
existing instanceof DeferredTypeEvidence && existing.isOnlyNulls(),
);
if (conflict !== undefined) {
throw branchConflictError(conflict, row, "Struct");
}
}
private addSchemaField(
schema: Schema,
path: string[],
value: unknown,
row: number,
): void {
const field = fieldAtPath(schema, path);
if (field === undefined) {
throw new Error(
`Found field not in schema: ${path.join(".")} at row ${row}`,
);
}
if (DataType.isStruct(field.type)) {
assertStructValue(path, value, row);
}
// Nulls are kept as deferred evidence so that a struct field seen first as
// null can be replaced by a record in a later row, just as a record seen
// first already ignores later nulls.
const conflict = this.fields.set(
path,
DeferredTypeEvidence.from(value, row) ?? field.type,
(existing) =>
existing instanceof DeferredTypeEvidence && existing.isOnlyNulls(),
);
if (conflict !== undefined) {
throw branchConflictError(conflict, row, "Struct");
}
}
private updateInferredField(
path: string[],
value: unknown,
row: number,
current: FieldNode,
): void {
const newType = this.inferType(value, path);
const deferred = DeferredTypeEvidence.from(value, row);
if (current instanceof FieldTree) {
if (deferred?.isOnlyNulls()) {
return;
}
throw schemaInferenceError(
path,
row,
"Struct",
describeEvidence(newType ?? deferred),
);
}
if (current instanceof DeferredTypeEvidence) {
this.resolveDeferredField(path, row, current, newType, deferred);
return;
}
if (newType !== undefined) {
if (!inferredTypesEqual(current, newType)) {
throw schemaInferenceError(
path,
row,
describeEvidence(current),
describeEvidence(newType),
);
}
return;
}
if (deferred === undefined || !deferred.matches(current)) {
throw schemaInferenceError(
path,
row,
describeEvidence(current),
describeEvidence(deferred),
);
}
}
private resolveDeferredField(
path: string[],
row: number,
current: DeferredTypeEvidence,
newType: DataType | undefined,
deferred: DeferredTypeEvidence | undefined,
): void {
if (newType !== undefined) {
if (!current.matches(newType)) {
throw schemaInferenceError(
path,
row,
current.describe(),
describeEvidence(newType),
);
}
this.fields.set(path, newType);
return;
}
if (deferred !== undefined) {
this.fields.set(path, current.merge(deferred));
return;
}
throw schemaInferenceError(
path,
row,
current.describe(),
describeEvidence(newType),
);
}
private inferType(value: unknown, path: string[]): DataType | undefined {
if (typeof value === "bigint") {
return new Int64();
}
if (typeof value === "number") {
return new Float64();
}
if (typeof value === "string") {
return this.options.dictionaryEncodeStrings
? new Dictionary(new Utf8(), new Int32())
: new Utf8();
}
if (typeof value === "boolean") {
return new Bool();
}
if (value instanceof Buffer) {
return new Binary();
}
if (ArrayBuffer.isView(value) && !(value instanceof DataView)) {
const typedArray = typedArrayToArrowType(value);
return typedArray === undefined
? undefined
: new FixedSizeList(
typedArray.length,
new Field("item", typedArray.elementType, true),
);
}
if (!Array.isArray(value) || value.length === 0) {
return undefined;
}
const configuredVector =
path.length === 1 ? this.options.vectorColumns[path[0]] : undefined;
if (configuredVector !== undefined) {
return new FixedSizeList(
value.length,
new Field("item", sanitizeType(configuredVector.type), true),
);
}
const itemType = this.inferArrayItemType(value, path);
if (itemType === undefined) {
return undefined;
}
return nameSuggestsVectorColumn(path[path.length - 1])
? new FixedSizeList(value.length, new Field("item", new Float32(), true))
: new List(new Field("item", itemType, true));
}
private inferArrayItemType(
values: unknown[],
path: string[],
): DataType | undefined {
let itemType: DataType | undefined;
const deferredItems: unknown[] = [];
for (const value of values) {
const candidate = this.inferType(value, path);
if (candidate === undefined) {
if (!isDeferredValue(value)) {
return undefined;
}
deferredItems.push(value);
} else if (itemType === undefined) {
itemType = candidate;
} else if (!inferredTypesEqual(itemType, candidate)) {
return undefined;
}
}
if (itemType === undefined) {
return undefined;
}
return deferredItems.every((value) =>
deferredValueMatchesType(value, itemType),
)
? itemType
: undefined;
}
}
/** Nulls and empty/all-null lists that do not determine a type by themselves. */
class DeferredTypeEvidence {
private constructor(
private readonly values: Array<{ value: unknown; row: number }>,
) {}
static from(value: unknown, row: number): DeferredTypeEvidence | undefined {
return isDeferredValue(value)
? new DeferredTypeEvidence([{ value, row }])
: undefined;
}
isOnlyNulls(): boolean {
return this.values.every(({ value }) => value == null);
}
matches(type: DataType): boolean {
return this.values.every(({ value }) =>
deferredValueMatchesType(value, type),
);
}
merge(other: DeferredTypeEvidence): DeferredTypeEvidence {
return new DeferredTypeEvidence([...this.values, ...other.values]);
}
describe(): string {
const list = this.values.find(({ value }) => Array.isArray(value));
return list === undefined
? "null"
: `List[${(list.value as unknown[]).length}]`;
}
firstRow(): number {
return this.values[0].row;
}
}
type FieldNode = DataType | DeferredTypeEvidence | FieldTree;
type LeafNode = Exclude<FieldNode, FieldTree>;
type FieldConflict = { path: string[]; value: FieldNode };
/** Nested field state, kept separate from Arrow's eventual Struct types. */
class FieldTree {
private readonly children = new Map<string, FieldNode>();
get(path: string[]): FieldNode | undefined {
let current: FieldNode = this;
for (const part of path) {
if (!(current instanceof FieldTree)) {
return undefined;
}
const child = current.children.get(part);
if (child === undefined) {
return undefined;
}
current = child;
}
return current;
}
set(
path: string[],
value: LeafNode,
canReplaceLeaf: (value: LeafNode) => boolean = () => false,
): FieldConflict | undefined {
let branch: FieldTree = this;
for (const [index, part] of path.slice(0, -1).entries()) {
const child = branch.children.get(part);
if (child === undefined || (isLeaf(child) && canReplaceLeaf(child))) {
const nextBranch = new FieldTree();
branch.children.set(part, nextBranch);
branch = nextBranch;
} else if (child instanceof FieldTree) {
branch = child;
} else {
return { path: path.slice(0, index + 1), value: child };
}
}
const name = path[path.length - 1];
const current = branch.children.get(name);
if (current instanceof FieldTree) {
return { path, value: current };
}
branch.children.set(name, value);
return undefined;
}
entries(): IterableIterator<[string, FieldNode]> {
return this.children.entries();
}
has(name: string): boolean {
return this.children.has(name);
}
}
function isLeaf(value: FieldNode): value is LeafNode {
return !(value instanceof FieldTree);
}
function fieldsFromTree(tree: FieldTree, path: string[] = []): Field[] {
const fields: Field[] = [];
for (const [name, value] of tree.entries()) {
if (value instanceof FieldTree) {
fields.push(
new Field(
name,
new Struct(fieldsFromTree(value, [...path, name])),
true,
),
);
} else if (value instanceof DeferredTypeEvidence) {
throw typeInferenceError([...path, name], value.firstRow());
} else {
fields.push(new Field(name, value, true));
}
}
return fields;
}
function matchingFields(fields: Field[], tree: FieldTree): Field[] {
const matches: Field[] = [];
for (const field of fields) {
if (!tree.has(field.name)) {
continue;
}
const value = tree.get([field.name]);
if (value instanceof FieldTree) {
const struct = field.type as Struct;
matches.push(
new Field(
field.name,
new Struct(matchingFields(struct.children, value)),
field.nullable,
field.metadata,
),
);
} else {
matches.push(field);
}
}
return matches;
}
function* recordPathsAndValues(
record: Record<string, unknown>,
path: string[] = [],
): Generator<[string[], unknown]> {
for (const [name, value] of Object.entries(record)) {
if (isRecord(value)) {
yield* recordPathsAndValues(value, [...path, name]);
} else if (value !== undefined) {
yield [[...path, name], value];
}
}
}
function isRecord(value: unknown): value is Record<string, unknown> {
if (typeof value !== "object" || value === null) {
return false;
}
const prototype = Object.getPrototypeOf(value);
return (
(prototype === Object.prototype || prototype === null) &&
Object.keys(value).length > 0
);
}
function isUnsupportedObject(value: unknown): boolean {
if (
typeof value !== "object" ||
value === null ||
Array.isArray(value) ||
isRecord(value)
) {
return false;
}
// Preserve the object types already passed through to Arrow. Other objects
// must not disappear while walking struct fields.
return !(
value instanceof RegExp ||
value instanceof Date ||
value instanceof Set ||
value instanceof Map ||
Buffer.isBuffer(value) ||
ArrayBuffer.isView(value)
);
}
function assertSupportedValue(
value: unknown,
field: string,
row: number,
): void {
if (Array.isArray(value)) {
for (const [index, item] of value.entries()) {
assertSupportedValue(item, `${field}[${index}]`, row);
}
} else if (isRecord(value)) {
for (const [name, child] of Object.entries(value)) {
assertSupportedValue(child, `${field}.${name}`, row);
}
} else if (isUnsupportedObject(value)) {
throw new Error(
`Unsupported object value for field ${field} at row ${row}.`,
);
}
}
/**
* Rejects primitives, arrays, and typed arrays given for a struct field, which
* would otherwise be written as a struct with every child null.
*/
function assertStructValue(path: string[], value: unknown, row: number): void {
if (
value == null ||
(typeof value === "object" &&
!Array.isArray(value) &&
!ArrayBuffer.isView(value))
) {
return;
}
const kind =
typeof value === "object" ? value.constructor.name : typeof value;
throw new Error(
`Expected a struct value for field ${path.join(".")} at row ${row}, got ${kind}.`,
);
}
function fieldAtPath(schema: Schema, path: string[]): Field | undefined {
let fields = schema.fields;
let field: Field | undefined;
for (const [index, name] of path.entries()) {
field = fields.find((candidate) => candidate.name === name);
if (field === undefined || index === path.length - 1) {
return field;
}
if (!DataType.isStruct(field.type)) {
return undefined;
}
fields = field.type.children;
}
return field;
}
function isDeferredValue(value: unknown): boolean {
return (
value == null || (Array.isArray(value) && value.every(isDeferredValue))
);
}
function deferredValueMatchesType(value: unknown, type: DataType): boolean {
if (value == null) {
return true;
}
if (!Array.isArray(value)) {
return false;
}
if (DataType.isList(type)) {
return value.every((item) =>
deferredValueMatchesType(item, type.valueType),
);
}
if (DataType.isFixedSizeList(type)) {
return (
value.length === type.listSize &&
value.every((item) => deferredValueMatchesType(item, type.valueType))
);
}
return false;
}
function inferredTypesEqual(current: DataType, candidate: DataType): boolean {
if (DataType.isDictionary(current)) {
return (
DataType.isDictionary(candidate) &&
current.isOrdered === candidate.isOrdered &&
inferredTypesEqual(current.indices, candidate.indices) &&
inferredTypesEqual(current.dictionary, candidate.dictionary)
);
}
if (DataType.isList(current)) {
return (
DataType.isList(candidate) &&
current.valueField.name === candidate.valueField.name &&
current.valueField.nullable === candidate.valueField.nullable &&
inferredTypesEqual(current.valueType, candidate.valueType)
);
}
if (DataType.isFixedSizeList(current)) {
return (
DataType.isFixedSizeList(candidate) &&
current.listSize === candidate.listSize &&
current.valueField.name === candidate.valueField.name &&
current.valueField.nullable === candidate.valueField.nullable &&
inferredTypesEqual(current.valueType, candidate.valueType)
);
}
return arrowUtil.compareTypes(current, candidate);
}
function describeEvidence(
evidence: DataType | DeferredTypeEvidence | undefined,
): string {
if (evidence === undefined) {
return "an unsupported value";
}
return evidence instanceof DeferredTypeEvidence
? evidence.describe()
: evidence.toString();
}
function branchConflictError(
conflict: FieldConflict,
row: number,
candidate: string,
): Error {
return schemaInferenceError(
conflict.path,
row,
conflict.value instanceof FieldTree
? "Struct"
: describeEvidence(conflict.value),
candidate,
);
}
function schemaInferenceError(
path: string[],
row: number,
currentType: string,
newType: string,
): Error {
return new Error(
`Failed to infer schema for data. Previously inferred type ${currentType} ` +
`but found ${newType} for field ${path.join(".")} at row ${row}. ` +
"Consider providing an explicit schema.",
);
}
function typeInferenceError(path: string[], row: number): Error {
return new Error(
`Failed to infer data type for field ${path.join(".")} at row ${row}. ` +
"Consider providing an explicit schema.",
);
}
function nameSuggestsVectorColumn(name: string): boolean {
const normalized = name.toLowerCase();
return normalized.includes("vector") || normalized.includes("embedding");
}
nodejs/lancedb/table.ts
+982 -50
View File
File diff suppressed because it is too large Load Diff
nodejs/lancedb/util.ts
+8
View File
@@ -15,6 +15,14 @@ export function toSQL(value: IntoSql): string {
if (typeof value === "string") {
return `'${value.replace(/'/g, "''")}'`;
} else if (typeof value === "number") {
// toString() gives "NaN" / "Infinity", which SQL reads as column names.
if (Number.isNaN(value)) {
return "CAST('NaN' AS DOUBLE)";
} else if (!Number.isFinite(value)) {
return value > 0
? "CAST('Infinity' AS DOUBLE)"
: "CAST('-Infinity' AS DOUBLE)";
}
return value.toString();
} else if (typeof value === "boolean") {
return value ? "TRUE" : "FALSE";
nodejs/lancedb/view.ts
+48
View File
@@ -0,0 +1,48 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { Schema, tableFromIPC } from "apache-arrow";
import { ViewDescription as NativeViewDescription } from "./native";
/**
* What a database records about one view.
*
* A view holds no rows: it stores the statement that defines it and the
* schema that statement resolved to, so a reader sees its sources as they
* are at read time.
*/
export interface ViewDescription {
/** The view's name within its namespace. */
name: string;
/** The namespace holding the view; empty is the root namespace. */
namespacePath: string[];
/** The defining query, as the database stores it. */
query: string;
/** The database that unqualified table names in {@link query} resolve against. */
defaultDatabase: string;
/**
* The namespace path those unqualified names resolve against; empty is the
* root namespace.
*
* Recorded with the view because it outlives the session that declared it:
* a reader resolving the query against its own default namespace could read
* a different table than the view was defined over.
*/
defaultNamespacePath: string[];
/** The schema the defining query resolved to when the view was created. */
schema: Schema;
}
/** Decode the schema the binding hands over as an Arrow IPC file. */
export function viewDescriptionFromNative(
view: NativeViewDescription,
): ViewDescription {
return {
name: view.name,
namespacePath: view.namespacePath,
query: view.query,
defaultDatabase: view.defaultDatabase,
defaultNamespacePath: view.defaultNamespacePath,
schema: tableFromIPC(view.schema).schema,
};
}
nodejs/npm/darwin-arm64/package.json
+5 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-darwin-arm64",
"version": "0.21.4-beta.1",
"version": "0.41.0-beta.13",
"os": ["darwin"],
"cpu": ["arm64"],
"main": "lancedb.darwin-arm64.node",
@@ -8,5 +8,9 @@
"license": "Apache-2.0",
"engines": {
"node": ">= 18"
},
"repository": {
"type": "git",
"url": "https://github.com/lancedb/lancedb"
}
}
nodejs/npm/darwin-x64/README.md
-3
View File
@@ -1,3 +0,0 @@
# `@lancedb/lancedb-darwin-x64`
This is the **x86_64-apple-darwin** binary for `@lancedb/lancedb`
nodejs/npm/darwin-x64/package.json
-12
View File
@@ -1,12 +0,0 @@
{
"name": "@lancedb/lancedb-darwin-x64",
"version": "0.21.4-beta.1",
"os": ["darwin"],
"cpu": ["x64"],
"main": "lancedb.darwin-x64.node",
"files": ["lancedb.darwin-x64.node"],
"license": "Apache-2.0",
"engines": {
"node": ">= 18"
}
}
nodejs/npm/linux-arm64-gnu/package.json
+6 -2
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-gnu",
"version": "0.21.4-beta.1",
"version": "0.41.0-beta.13",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-gnu.node",
@@ -9,5 +9,9 @@
"engines": {
"node": ">= 18"
},
"libc": ["glibc"]
"libc": ["glibc"],
"repository": {
"type": "git",
"url": "https://github.com/lancedb/lancedb"
}
}
nodejs/npm/linux-arm64-musl/package.json
+6 -2
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-musl",
"version": "0.21.4-beta.1",
"version": "0.41.0-beta.13",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-musl.node",
@@ -9,5 +9,9 @@
"engines": {
"node": ">= 18"
},
"libc": ["musl"]
"libc": ["musl"],
"repository": {
"type": "git",
"url": "https://github.com/lancedb/lancedb"
}
}
nodejs/npm/linux-x64-gnu/package.json
+6 -2
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-gnu",
"version": "0.21.4-beta.1",
"version": "0.41.0-beta.13",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-gnu.node",
@@ -9,5 +9,9 @@
"engines": {
"node": ">= 18"
},
"libc": ["glibc"]
"libc": ["glibc"],
"repository": {
"type": "git",
"url": "https://github.com/lancedb/lancedb"
}
}
nodejs/npm/linux-x64-musl/package.json
+6 -2
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-musl",
"version": "0.21.4-beta.1",
"version": "0.41.0-beta.13",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-musl.node",
@@ -9,5 +9,9 @@
"engines": {
"node": ">= 18"
},
"libc": ["musl"]
"libc": ["musl"],
"repository": {
"type": "git",
"url": "https://github.com/lancedb/lancedb"
}
}
nodejs/npm/win32-arm64-msvc/package.json
+5 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-arm64-msvc",
"version": "0.21.4-beta.1",
"version": "0.41.0-beta.13",
"os": [
"win32"
],
@@ -14,5 +14,9 @@
"license": "Apache-2.0",
"engines": {
"node": ">= 18"
},
"repository": {
"type": "git",
"url": "https://github.com/lancedb/lancedb"
}
}
nodejs/npm/win32-x64-msvc/package.json
+5 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-x64-msvc",
"version": "0.21.4-beta.1",
"version": "0.41.0-beta.13",
"os": ["win32"],
"cpu": ["x64"],
"main": "lancedb.win32-x64-msvc.node",
@@ -8,5 +8,9 @@
"license": "Apache-2.0",
"engines": {
"node": ">= 18"
},
"repository": {
"type": "git",
"url": "https://github.com/lancedb/lancedb"
}
}
nodejs/package-lock.json
Generated
-9882
View File
File diff suppressed because it is too large Load Diff
nodejs/package.json
+48 -34
View File
@@ -11,7 +11,7 @@
"ann"
],
"private": false,
"version": "0.21.4-beta.1",
"version": "0.41.0-beta.13",
"main": "dist/index.js",
"exports": {
".": "./dist/index.js",
@@ -21,61 +21,68 @@
},
"types": "dist/index.d.ts",
"napi": {
"name": "lancedb",
"triples": {
"defaults": false,
"additional": [
"x86_64-apple-darwin",
"aarch64-apple-darwin",
"x86_64-unknown-linux-gnu",
"aarch64-unknown-linux-gnu",
"x86_64-unknown-linux-musl",
"aarch64-unknown-linux-musl",
"x86_64-pc-windows-msvc",
"aarch64-pc-windows-msvc"
]
}
"binaryName": "lancedb",
"targets": [
"aarch64-apple-darwin",
"x86_64-unknown-linux-gnu",
"aarch64-unknown-linux-gnu",
"x86_64-unknown-linux-musl",
"aarch64-unknown-linux-musl",
"x86_64-pc-windows-msvc",
"aarch64-pc-windows-msvc"
]
},
"license": "Apache-2.0",
"repository": {
"type": "git",
"url": "https://github.com/lancedb/lancedb"
},
"devDependencies": {
"@aws-sdk/client-dynamodb": "^3.33.0",
"@aws-sdk/client-kms": "^3.33.0",
"@aws-sdk/client-s3": "^3.33.0",
"@aws-sdk/client-dynamodb": "3.1003.0",
"@aws-sdk/client-kms": "3.1003.0",
"@aws-sdk/client-s3": "3.1003.0",
"@biomejs/biome": "^1.7.3",
"@jest/globals": "^29.7.0",
"@napi-rs/cli": "^2.18.3",
"@napi-rs/cli": "3.7.0",
"@opentelemetry/sdk-metrics": "^2.10.0",
"@types/axios": "^0.14.0",
"@types/jest": "^29.1.2",
"@types/node": "^22.7.4",
"@types/node": "22.7.4",
"@types/tmp": "^0.2.6",
"apache-arrow": "21.2.0",
"apache-arrow-15": "npm:apache-arrow@15.0.0",
"apache-arrow-16": "npm:apache-arrow@16.0.0",
"apache-arrow-17": "npm:apache-arrow@17.0.0",
"apache-arrow-18": "npm:apache-arrow@18.0.0",
"apache-arrow-19": "npm:apache-arrow@19.0.1",
"apache-arrow-20": "npm:apache-arrow@20.0.0",
"apache-arrow-21": "npm:apache-arrow@21.2.0",
"eslint": "^8.57.0",
"jest": "^29.7.0",
"shx": "^0.3.4",
"tmp": "^0.2.3",
"tmp": "^0.2.7",
"ts-jest": "^29.1.2",
"typedoc": "^0.26.4",
"typedoc-plugin-markdown": "^4.2.1",
"typescript": "^5.5.4",
"typedoc": "0.26.4",
"typedoc-plugin-markdown": "4.2.1",
"typescript": "5.5.4",
"typescript-eslint": "^7.1.0"
},
"ava": {
"timeout": "3m"
},
"engines": {
"node": ">= 18"
"node": ">= 22"
},
"packageManager": "pnpm@11.1.1",
"cpu": ["x64", "arm64"],
"os": ["darwin", "linux", "win32"],
"scripts": {
"artifacts": "napi artifacts",
"build:debug": "napi build --platform --no-const-enum --dts ../lancedb/native.d.ts --js ../lancedb/native.js lancedb",
"build:release": "napi build --platform --no-const-enum --release --dts ../lancedb/native.d.ts --js ../lancedb/native.js dist/",
"build": "npm run build:debug && npm run tsc && shx cp lancedb/*.node dist/",
"build-release": "npm run build:release && npm run tsc",
"build:debug": "napi build --platform --dts ../lancedb/native.d.ts --js ../lancedb/native.js --output-dir lancedb",
"postbuild:debug": "shx mkdir -p dist && shx cp lancedb/*.node dist/ && node -e \"require('fs').writeFileSync('dist/package.json', JSON.stringify({name:'@lancedb/lancedb',type:'commonjs'}))\"",
"build:release": "napi build --platform --release --dts ../lancedb/native.d.ts --js ../lancedb/native.js --output-dir dist",
"build": "pnpm build:debug && pnpm tsc",
"build-release": "pnpm build:release && pnpm tsc",
"tsc": "tsc -b",
"posttsc": "shx cp lancedb/native.d.ts dist/native.d.ts",
"lint-ci": "biome ci .",
@@ -85,18 +92,25 @@
"lint-fix": "biome check --write . && biome format --write .",
"prepublishOnly": "napi prepublish -t npm",
"test": "jest --verbose",
"integration": "S3_TEST=1 npm run test",
"universal": "napi universal",
"integration": "S3_TEST=1 pnpm test",
"universal": "napi universalize",
"version": "napi version"
},
"dependencies": {
"@opentelemetry/api": "^1.9.0",
"reflect-metadata": "^0.2.2"
},
"optionalDependencies": {
"@huggingface/transformers": "^3.0.2",
"openai": "^4.29.2"
"@huggingface/transformers": "3.0.2",
"openai": "4.29.2"
},
"peerDependencies": {
"apache-arrow": ">=15.0.0 <=18.1.0"
"@types/node": ">=22",
"apache-arrow": ">=15.0.0 <22.0.0"
},
"peerDependenciesMeta": {
"@types/node": {
"optional": true
}
}
}
nodejs/pnpm-lock.yaml
Generated
+7610
View File
File diff suppressed because it is too large Load Diff
nodejs/pnpm-workspace.yaml
+56
View File
@@ -0,0 +1,56 @@
# Flat node_modules layout. The @napi-rs/cli build step fails to locate
# the cdylib artifact under pnpm's isolated layout; the hoisted linker
# mirrors npm's structure and unblocks the native build.
nodeLinker: hoisted
# Block resolution of versions less than 24h old (Shai-Hulud window).
# This is the pnpm 11 default but pinned here so it's visible to
# reviewers and survives a future pnpm major flipping the default.
minimumReleaseAge: 1440
# Fail install if a transitive dep tries to run an unapproved script.
strictDepBuilds: true
allowBuilds:
'@biomejs/biome': true
onnxruntime-node: true
protobufjs: true
sharp: true
minimumReleaseAgeExclude:
- protobufjs@7.5.8
- tmp@0.2.6
- form-data@4.0.6
- tar@7.5.16
- markdown-it@14.1.2
- linkify-it@5.0.1
- js-yaml@3.15.0
- js-yaml@4.1.2
- protobufjs@7.6.1
- protobufjs@7.6.3
- '@babel/core@7.29.1'
- axios@1.18.0
- brace-expansion@2.1.2
- brace-expansion@1.1.16
- js-yaml@4.3.0
- tar@7.5.18
- tar@7.5.19
- tar@7.5.17
- protobufjs@7.6.5
- linkify-it@5.0.2
- sharp@0.35.0
- brace-expansion@1.1.17
- brace-expansion@2.1.3
- brace-expansion@2.1.4
- brace-expansion@1.1.18
- js-yaml@3.15.1
- js-yaml@4.3.1
- tar@7.5.21
- '@opentelemetry/core@2.8.0'
# @huggingface/transformers pins sharp ^0.33.5 and no released version has moved
# past ^0.34.5, all of which inherit the libvips CVEs in GHSA-f88m-g3jw-g9cj.
# Force the patched line. sharp is only reached by transformers' image pipeline,
# which LanceDB's text embedding function never uses.
overrides:
sharp: ^0.35.4
nodejs/src/blob.rs
+218
View File
@@ -0,0 +1,218 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::future::Future;
use std::ops::Range;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use arrow_array::{Array, LargeBinaryArray};
use futures::future::abortable;
use lancedb::blob::BlobFile as LanceBlobFile;
use napi::Env;
use napi::bindgen_prelude::*;
use napi_derive::napi;
use crate::error::convert_error;
#[napi(object)]
pub struct BlobRange {
pub start: BigInt,
pub end: BigInt,
}
#[napi]
pub struct BlobFile {
inner: Arc<LanceBlobFile>,
closed: AtomicBool,
}
impl BlobFile {
pub(crate) fn new(inner: LanceBlobFile) -> Self {
Self {
inner: Arc::new(inner),
closed: AtomicBool::new(false),
}
}
fn ensure_open(&self) -> napi::Result<()> {
if self.closed.load(Ordering::Acquire) {
Err(napi::Error::from_reason("blob file is already closed"))
} else {
Ok(())
}
}
}
#[napi]
impl BlobFile {
#[napi]
pub fn size(&self) -> BigInt {
BigInt::from(self.inner.size())
}
#[napi]
pub fn read<'env>(
&self,
env: &'env Env,
max_bytes: Option<BigInt>,
signal: Option<AbortSignal>,
) -> napi::Result<PromiseRaw<'env, Buffer>> {
self.ensure_open()?;
let max_bytes = max_bytes
.map(|max_bytes| {
usize::try_from(parse_u64(max_bytes, "maxBytes")?)
.map_err(|_| napi::Error::from_reason("maxBytes is too large"))
})
.transpose()?;
let inner = self.inner.clone();
spawn_abortable(env, signal, async move {
let bytes = match max_bytes {
None => inner.read().await,
Some(max_bytes) => inner.read_up_to(max_bytes).await,
}
.map_err(|err| convert_error(&err))?;
Ok(Buffer::from(bytes.as_ref()))
})
}
#[napi]
pub fn read_range<'env>(
&self,
env: &'env Env,
start: BigInt,
end: BigInt,
signal: Option<AbortSignal>,
) -> napi::Result<PromiseRaw<'env, Buffer>> {
self.ensure_open()?;
let range = bigint_range(start, end)?;
let inner = self.inner.clone();
spawn_abortable(env, signal, async move {
let bytes = inner
.read_range(range)
.await
.map_err(|err| convert_error(&err))?;
Ok(Buffer::from(bytes.as_ref()))
})
}
#[napi]
pub fn read_ranges<'env>(
&self,
env: &'env Env,
ranges: Vec<BlobRange>,
signal: Option<AbortSignal>,
) -> napi::Result<PromiseRaw<'env, Vec<Buffer>>> {
self.ensure_open()?;
let ranges = ranges
.into_iter()
.map(|range| bigint_range(range.start, range.end))
.collect::<napi::Result<Vec<_>>>()?;
let inner = self.inner.clone();
spawn_abortable(env, signal, async move {
let buffers = inner
.read_ranges(&ranges)
.await
.map_err(|err| convert_error(&err))?;
Ok(buffers
.iter()
.map(|bytes| Buffer::from(bytes.as_ref()))
.collect())
})
}
#[napi]
pub async fn seek(&self, position: BigInt) -> napi::Result<()> {
self.ensure_open()?;
let position = parse_u64(position, "position")?;
self.inner
.seek(position)
.await
.map_err(|err| convert_error(&err))
}
#[napi]
pub async fn tell(&self) -> napi::Result<BigInt> {
self.ensure_open()?;
let position = self.inner.tell().await.map_err(|err| convert_error(&err))?;
Ok(BigInt::from(position))
}
#[napi]
pub async fn close(&self) -> napi::Result<()> {
if self.closed.swap(true, Ordering::AcqRel) {
return Ok(());
}
self.inner.close().await.map_err(|err| convert_error(&err))
}
#[napi]
pub fn is_closed(&self) -> bool {
self.closed.load(Ordering::Acquire)
}
}
/// Runs `fut` on the napi runtime. Aborting `signal` drops the future, which
/// cancels in-flight reads.
pub fn spawn_abortable<'env, T>(
env: &'env Env,
signal: Option<AbortSignal>,
fut: impl Future<Output = napi::Result<T>> + Send + 'static,
) -> napi::Result<PromiseRaw<'env, T>>
where
T: ToNapiValue + Send + 'static,
{
let (fut, handle) = abortable(fut);
if let Some(signal) = signal {
signal.on_abort(move || handle.abort());
}
env.spawn_future(async move {
fut.await
.unwrap_or_else(|_| Err(napi::Error::from_reason("blob read aborted")))
})
}
fn bigint_range(start: BigInt, end: BigInt) -> napi::Result<Range<u64>> {
let start = parse_u64(start, "start")?;
let end = parse_u64(end, "end")?;
if start > end {
return Err(napi::Error::from_reason(format!(
"invalid blob range: start ({start}) > end ({end})"
)));
}
Ok(start..end)
}
fn parse_u64(value: BigInt, name: &str) -> napi::Result<u64> {
let (negative, value, lossless) = value.get_u64();
if negative {
return Err(napi::Error::from_reason(format!(
"{name} cannot be negative"
)));
}
if !lossless {
return Err(napi::Error::from_reason(format!(
"{name} is too large to fit in u64"
)));
}
Ok(value)
}
pub fn parse_row_ids(row_ids: Vec<BigInt>) -> napi::Result<Vec<u64>> {
row_ids
.into_iter()
.map(|id| parse_u64(id, "row id"))
.collect()
}
pub fn copy_blob_buffers(array: LargeBinaryArray) -> Vec<Option<Buffer>> {
(0..array.len())
.map(|i| {
if array.is_null(i) {
None
} else {
Some(Buffer::from(array.value(i).to_vec()))
}
})
.collect()
}
nodejs/src/catalog.rs
+146
View File
@@ -0,0 +1,146 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::sync::{Arc, Mutex};
use std::time::Duration;
use futures::{StreamExt, future::poll_fn};
use lancedb::catalog::{CatalogConnection, CreateDatabaseRequest, DropDatabaseRequest};
use napi::bindgen_prelude::*;
use napi_derive::napi;
use crate::connection::Connection;
use crate::error::NapiErrorExt;
use crate::header::JsHeaderProvider;
use crate::remote::{ClientConfig, OAuthConfig};
#[napi(object)]
pub struct CatalogOptions {
pub api_key: Option<String>,
pub client_config: Option<ClientConfig>,
/// SQL service endpoint inherited by database connections.
pub sql_host_override: Option<String>,
pub read_consistency_interval: Option<f64>,
pub oauth_config: Option<OAuthConfig>,
}
/// A lazy iterator over database names with inspectable pagination state.
#[napi]
pub struct DatabaseNames {
inner: Mutex<lancedb::catalog::DatabaseNames>,
next_lock: futures::lock::Mutex<()>,
}
#[napi]
impl DatabaseNames {
/// Number of names cached without another REST request.
#[napi]
pub fn num_page_results(&self) -> u32 {
self.inner.lock().unwrap().num_page_results() as u32
}
/// Continuation token for the next REST request.
#[napi]
pub fn page_token(&self) -> Option<String> {
self.inner.lock().unwrap().page_token().map(str::to_owned)
}
/// Fetch the next name, returning None when exhausted.
#[napi]
pub async fn next(&self) -> Result<Option<String>> {
// Serialize advances but allow synchronous inspection between polls.
let _guard = self.next_lock.lock().await;
poll_fn(|cx| self.inner.lock().unwrap().poll_next_unpin(cx))
.await
.transpose()
.default_error()
}
}
#[napi]
pub struct Catalog {
inner: CatalogConnection,
}
#[napi]
impl Catalog {
#[napi(factory)]
pub async fn new(
endpoint: String,
options: CatalogOptions,
header_provider: Option<&JsHeaderProvider>,
) -> Result<Self> {
let mut builder = lancedb::connect_catalog(endpoint);
if let Some(key) = options.api_key {
builder = builder.api_key(key);
}
let mut config: lancedb::remote::ClientConfig =
options.client_config.unwrap_or_default().into();
if let Some(provider) = header_provider {
config.header_provider = Some(Arc::new(provider.clone()));
}
builder = builder.client_config(config);
if let Some(endpoint) = options.sql_host_override {
builder = builder.sql_host_override(endpoint);
}
if let Some(interval) = options.read_consistency_interval {
let interval = Duration::try_from_secs_f64(interval).map_err(|err| {
Error::from_reason(format!("Invalid read consistency interval: {err}"))
})?;
builder = builder.read_consistency_interval(interval);
}
if let Some(oauth) = options.oauth_config {
builder = builder.oauth_config(oauth.try_into().default_error()?);
}
Ok(Self {
inner: builder.execute().await.default_error()?,
})
}
#[napi(getter)]
pub fn uri(&self) -> String {
self.inner.uri().to_string()
}
#[napi]
pub async fn create_database(
&self,
name: String,
exist_ok: Option<bool>,
) -> Result<Connection> {
self.inner
.create_database(CreateDatabaseRequest::new(name).exist_ok(exist_ok.unwrap_or(false)))
.await
.map(Connection::inner_new)
.default_error()
}
#[napi]
pub async fn connect_database(&self, name: String) -> Result<Connection> {
self.inner
.connect_database(name)
.await
.map(Connection::inner_new)
.default_error()
}
#[napi]
pub async fn drop_database(&self, name: String, ignore_missing: Option<bool>) -> Result<()> {
self.inner
.drop_database(
DropDatabaseRequest::new(name).ignore_missing(ignore_missing.unwrap_or(false)),
)
.await
.default_error()
}
#[napi]
pub fn list_databases(
&self,
page_token: Option<String>,
page_limit: Option<u32>,
) -> DatabaseNames {
DatabaseNames {
inner: Mutex::new(self.inner.list_databases(page_token, page_limit)),
next_lock: futures::lock::Mutex::new(()),
}
}
}
nodejs/src/connection.rs
+589 -18
View File
@@ -2,15 +2,23 @@
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::collections::HashMap;
use std::sync::Arc;
use lancedb::database::CreateTableMode;
use lancedb::database::{CreateTableMode, Database};
use napi::bindgen_prelude::*;
use napi_derive::*;
use crate::error::NapiErrorExt;
use crate::table::Table;
use crate::ConnectNamespaceOptions;
use crate::ConnectionOptions;
use lancedb::connection::{ConnectBuilder, Connection as LanceDBConnection};
use crate::error::NapiErrorExt;
use crate::header::JsHeaderProvider;
use crate::table::Table;
use lancedb::connection::{ConnectBuilder, Connection as LanceDBConnection, connect_namespace};
use lance_namespace::models::{
CreateNamespaceRequest, DescribeNamespaceRequest, DropNamespaceRequest, ListNamespacesRequest,
ListTablesRequest,
};
use lancedb::ipc::{ipc_file_to_batches, ipc_file_to_schema};
#[napi]
@@ -18,6 +26,61 @@ pub struct Connection {
inner: Option<LanceDBConnection>,
}
#[napi(object)]
pub struct DescribeNamespaceResponse {
pub properties: Option<HashMap<String, String>>,
}
#[napi(object)]
pub struct ListNamespacesResponse {
pub namespaces: Vec<String>,
pub page_token: Option<String>,
}
#[napi(object)]
pub struct ListTablesResponse {
pub tables: Vec<String>,
pub page_token: Option<String>,
}
#[napi(object)]
pub struct CreateNamespaceResponse {
pub properties: Option<HashMap<String, String>>,
pub transaction_id: Option<String>,
}
#[napi(object)]
pub struct DropNamespaceResponse {
pub properties: Option<HashMap<String, String>>,
pub transaction_id: Option<Vec<String>>,
}
/// What a database records about one view.
#[napi(object)]
pub struct ViewDescription {
pub name: String,
pub namespace_path: Vec<String>,
pub query: String,
pub default_database: String,
pub default_namespace_path: Vec<String>,
/// The view's schema as an empty Arrow IPC file, the way a table reports
/// its own.
pub schema: Buffer,
}
impl ViewDescription {
fn from_inner(view: lancedb::view::ViewDescription) -> napi::Result<Self> {
Ok(Self {
name: view.name,
namespace_path: view.namespace_path,
query: view.query,
default_database: view.default_database,
default_namespace_path: view.default_namespace_path,
schema: crate::util::schema_to_buffer(&view.schema)?,
})
}
}
impl Connection {
pub(crate) fn inner_new(inner: LanceDBConnection) -> Self {
Self { inner: Some(inner) }
@@ -39,13 +102,21 @@ impl Connection {
_ => Err(napi::Error::from_reason(format!("Invalid mode {}", mode))),
}
}
pub fn database(&self) -> napi::Result<Arc<dyn Database>> {
Ok(self.get_inner()?.database().clone())
}
}
#[napi]
impl Connection {
/// Create a new Connection instance from the given URI.
#[napi(factory)]
pub async fn new(uri: String, options: ConnectionOptions) -> napi::Result<Self> {
pub async fn new(
uri: String,
options: ConnectionOptions,
header_provider: Option<&JsHeaderProvider>,
) -> napi::Result<Self> {
let mut builder = ConnectBuilder::new(&uri);
if let Some(interval) = options.read_consistency_interval {
builder =
@@ -56,9 +127,29 @@ impl Connection {
builder = builder.storage_option(key, value);
}
}
if let Some(manifest_enabled) = options.manifest_enabled {
builder = builder.manifest_enabled(manifest_enabled);
}
if let Some(namespace_client_properties) = options.namespace_client_properties {
builder = builder.namespace_client_properties(namespace_client_properties);
}
// Create client config, optionally with header provider
let client_config = options.client_config.unwrap_or_default();
builder = builder.client_config(client_config.into());
let mut rust_config: lancedb::remote::ClientConfig = client_config.into();
if let Some(provider) = header_provider {
rust_config.header_provider =
Some(Arc::new(provider.clone()) as Arc<dyn lancedb::remote::HeaderProvider>);
}
builder = builder.client_config(rust_config);
if let Some(oauth_config) = options.oauth_config {
let config: lancedb::remote::oauth::OAuthConfig =
oauth_config.try_into().default_error()?;
builder = builder.oauth_config(config);
}
if let Some(api_key) = options.api_key {
builder = builder.api_key(&api_key);
@@ -81,6 +172,39 @@ impl Connection {
Ok(Self::inner_new(builder.execute().await.default_error()?))
}
/// Create a new Connection instance backed by a namespace implementation.
#[napi(factory)]
pub async fn new_with_namespace(
impl_name: String,
properties: HashMap<String, String>,
options: ConnectNamespaceOptions,
) -> napi::Result<Self> {
if impl_name.is_empty() {
return Err(napi::Error::from_reason(
"implName must be a non-empty string",
));
}
let mut builder = connect_namespace(&impl_name, properties);
if let Some(interval) = options.read_consistency_interval {
builder =
builder.read_consistency_interval(std::time::Duration::from_secs_f64(interval));
}
if let Some(storage_options) = options.storage_options {
for (key, value) in storage_options {
builder = builder.storage_option(key, value);
}
}
if let Some(namespace_client_properties) = options.namespace_client_properties {
builder = builder.namespace_client_properties(namespace_client_properties);
}
if let Some(session) = options.session {
builder = builder.session(session.inner.clone());
}
Ok(Self::inner_new(builder.execute().await.default_error()?))
}
#[napi]
pub fn display(&self) -> napi::Result<String> {
Ok(self.get_inner()?.to_string())
@@ -100,12 +224,12 @@ impl Connection {
#[napi(catch_unwind)]
pub async fn table_names(
&self,
namespace: Vec<String>,
namespace_path: Option<Vec<String>>,
start_after: Option<String>,
limit: Option<u32>,
) -> napi::Result<Vec<String>> {
let mut op = self.get_inner()?.table_names();
op = op.namespace(namespace);
op = op.namespace(namespace_path.unwrap_or_default());
if let Some(start_after) = start_after {
op = op.start_after(start_after);
}
@@ -115,6 +239,33 @@ impl Connection {
op.execute().await.default_error()
}
/// List a page of tables in the database.
#[napi(catch_unwind)]
pub async fn list_tables(
&self,
namespace_path: Option<Vec<String>>,
page_token: Option<String>,
limit: Option<u32>,
) -> napi::Result<ListTablesResponse> {
let request = ListTablesRequest {
// The root namespace is an empty path, not an absent one: a namespace-backed
// database rejects a request that names no namespace.
id: Some(namespace_path.unwrap_or_default()),
page_token,
limit: limit.map(|limit| i32::try_from(limit).unwrap_or(i32::MAX)),
..Default::default()
};
let response = self
.get_inner()?
.list_tables(request)
.await
.default_error()?;
Ok(ListTablesResponse {
tables: response.tables,
page_token: response.page_token,
})
}
/// Create table from a Apache Arrow IPC (file) buffer.
///
/// Parameters:
@@ -127,7 +278,7 @@ impl Connection {
name: String,
buf: Buffer,
mode: String,
namespace: Vec<String>,
namespace_path: Option<Vec<String>>,
storage_options: Option<HashMap<String, String>>,
) -> napi::Result<Table> {
let batches = ipc_file_to_batches(buf.to_vec())
@@ -135,7 +286,7 @@ impl Connection {
let mode = Self::parse_create_mode_str(&mode)?;
let mut builder = self.get_inner()?.create_table(&name, batches).mode(mode);
builder = builder.namespace(namespace);
builder = builder.namespace(namespace_path.unwrap_or_default());
if let Some(storage_options) = storage_options {
for (key, value) in storage_options {
@@ -152,7 +303,7 @@ impl Connection {
name: String,
schema_buf: Buffer,
mode: String,
namespace: Vec<String>,
namespace_path: Option<Vec<String>>,
storage_options: Option<HashMap<String, String>>,
) -> napi::Result<Table> {
let schema = ipc_file_to_schema(schema_buf.to_vec()).map_err(|e| {
@@ -164,7 +315,7 @@ impl Connection {
.create_empty_table(&name, schema)
.mode(mode);
builder = builder.namespace(namespace);
builder = builder.namespace(namespace_path.unwrap_or_default());
if let Some(storage_options) = storage_options {
for (key, value) in storage_options {
@@ -175,17 +326,183 @@ impl Connection {
Ok(Table::new(tbl))
}
#[napi(catch_unwind)]
pub async fn create_materialized_view(
&self,
name: String,
source: String,
projections: Option<Vec<Vec<String>>>,
filter: Option<String>,
limit: Option<i64>,
with_no_data: bool,
) -> napi::Result<Table> {
let mut builder = self.get_inner()?.create_materialized_view(name, source);
if let Some(projections) = projections {
let mut pairs = Vec::with_capacity(projections.len());
for pair in projections {
let [output, expression]: [String; 2] = pair.try_into().map_err(|_| {
napi::Error::from_reason("each projection must be an [output, expression] pair")
})?;
pairs.push((output, expression));
}
builder = builder.select(pairs);
}
if let Some(filter) = filter {
builder = builder.only_if(filter);
}
if let Some(limit) = limit {
let limit = u64::try_from(limit)
.map_err(|_| napi::Error::from_reason("limit must be a non-negative integer"))?;
builder = builder.limit(limit);
}
builder = builder.with_no_data(with_no_data);
let view = builder.execute().await.default_error()?;
Ok(Table::new(view.table().clone()))
}
#[napi(catch_unwind)]
pub async fn open_materialized_view(&self, name: String) -> napi::Result<Table> {
let view = self
.get_inner()?
.open_materialized_view(&name)
.await
.default_error()?;
Ok(Table::new(view.table().clone()))
}
#[napi(catch_unwind)]
pub fn list_materialized_views(
&self,
page_token: Option<String>,
page_limit: Option<u32>,
) -> napi::Result<crate::listing::NameListing> {
Ok(crate::listing::NameListing::new(
self.get_inner()?
.list_materialized_views(crate::listing::options(page_token, page_limit)),
))
}
/// Drop a materialized view.
#[napi(catch_unwind)]
pub async fn drop_materialized_view(
&self,
name: String,
namespace_path: Option<Vec<String>>,
) -> napi::Result<()> {
let ns = namespace_path.unwrap_or_default();
self.get_inner()?
.drop_materialized_view(&name, &ns)
.await
.default_error()
}
/// Create a view: a named query planned on every read.
#[napi(catch_unwind)]
pub async fn create_view(
&self,
name: String,
query: String,
namespace_path: Option<Vec<String>>,
) -> napi::Result<ViewDescription> {
let ns = namespace_path.unwrap_or_default();
let view = self
.get_inner()?
.create_view(&name, &query, &ns)
.await
.default_error()?;
ViewDescription::from_inner(view)
}
/// What the database records about one view.
#[napi(catch_unwind)]
pub async fn describe_view(
&self,
name: String,
namespace_path: Option<Vec<String>>,
) -> napi::Result<ViewDescription> {
let ns = namespace_path.unwrap_or_default();
let view = self
.get_inner()?
.describe_view(&name, &ns)
.await
.default_error()?;
ViewDescription::from_inner(view)
}
/// Drop a view and wait for its definition to be deleted. The tables it
/// reads are untouched.
#[napi(catch_unwind)]
pub async fn drop_view(
&self,
name: String,
namespace_path: Option<Vec<String>>,
) -> napi::Result<()> {
let ns = namespace_path.unwrap_or_default();
self.get_inner()?
.drop_view(&name, &ns)
.await
.default_error()
}
/// Start dropping a view and return the job deleting its definition.
#[napi(catch_unwind)]
pub async fn drop_view_async(
&self,
name: String,
namespace_path: Option<Vec<String>>,
) -> napi::Result<crate::job::Job> {
let ns = namespace_path.unwrap_or_default();
let job = self
.get_inner()?
.drop_view_async(&name, &ns)
.await
.default_error()?;
Ok(crate::job::Job::new(job))
}
/// The names of the views in one namespace.
#[napi(catch_unwind)]
pub fn list_views(
&self,
namespace_path: Option<Vec<String>>,
page_token: Option<String>,
page_limit: Option<u32>,
) -> napi::Result<crate::listing::NameListing> {
Ok(crate::listing::NameListing::new(
self.get_inner()?.list_views(
&namespace_path.unwrap_or_default(),
crate::listing::options(page_token, page_limit),
),
))
}
/// Start dropping a materialized view and return its cleanup job.
#[napi(catch_unwind)]
pub async fn drop_materialized_view_async(
&self,
name: String,
namespace_path: Option<Vec<String>>,
) -> napi::Result<crate::job::Job> {
let ns = namespace_path.unwrap_or_default();
let job = self
.get_inner()?
.drop_materialized_view_async(&name, &ns)
.await
.default_error()?;
Ok(crate::job::Job::new(job))
}
#[napi(catch_unwind)]
pub async fn open_table(
&self,
name: String,
namespace: Vec<String>,
namespace_path: Option<Vec<String>>,
storage_options: Option<HashMap<String, String>>,
index_cache_size: Option<u32>,
) -> napi::Result<Table> {
let mut builder = self.get_inner()?.open_table(&name);
builder = builder.namespace(namespace);
builder = builder.namespace(namespace_path.unwrap_or_default());
if let Some(storage_options) = storage_options {
for (key, value) in storage_options {
@@ -199,19 +516,273 @@ impl Connection {
Ok(Table::new(tbl))
}
#[napi(catch_unwind)]
pub async fn clone_table(
&self,
target_table_name: String,
source_uri: String,
target_namespace_path: Option<Vec<String>>,
source_version: Option<i64>,
source_tag: Option<String>,
is_shallow: bool,
) -> napi::Result<Table> {
let mut builder = self
.get_inner()?
.clone_table(&target_table_name, &source_uri);
builder = builder.target_namespace(target_namespace_path.unwrap_or_default());
if let Some(version) = source_version {
builder = builder.source_version(version as u64);
}
if let Some(tag) = source_tag {
builder = builder.source_tag(tag);
}
builder = builder.is_shallow(is_shallow);
let tbl = builder.execute().await.default_error()?;
Ok(Table::new(tbl))
}
/// Drop table with the name. Or raise an error if the table does not exist.
#[napi(catch_unwind)]
pub async fn drop_table(&self, name: String, namespace: Vec<String>) -> napi::Result<()> {
pub async fn drop_table(
&self,
name: String,
namespace_path: Option<Vec<String>>,
) -> napi::Result<()> {
let ns = namespace_path.unwrap_or_default();
self.get_inner()?
.drop_table(&name, &namespace)
.drop_table(&name, &ns)
.await
.default_error()
}
/// Start dropping a table and return its cleanup job.
#[napi(catch_unwind)]
pub async fn drop_all_tables(&self, namespace: Vec<String>) -> napi::Result<()> {
pub async fn drop_table_async(
&self,
name: String,
namespace_path: Option<Vec<String>>,
) -> napi::Result<crate::job::Job> {
let ns = namespace_path.unwrap_or_default();
let job = self
.get_inner()?
.drop_table_async(&name, &ns)
.await
.default_error()?;
Ok(crate::job::Job::new(job))
}
#[napi(catch_unwind)]
pub async fn drop_all_tables(&self, namespace_path: Option<Vec<String>>) -> napi::Result<()> {
let ns = namespace_path.unwrap_or_default();
self.get_inner()?.drop_all_tables(&ns).await.default_error()
}
/// Open a server-side job by id, returning a handle with its record
/// already populated. Rejects when the server has no such job.
///
/// The returned handle answers for its own state, specification, result,
/// failure and event history, so there is no separate connection-level
/// call for any of them.
#[napi(catch_unwind)]
pub async fn open_job(&self, job_id: String) -> napi::Result<crate::job::Job> {
let job = self.get_inner()?.open_job(&job_id).await.default_error()?;
Ok(crate::job::Job::new(job))
}
/// List server-side jobs across the database's tables.
#[napi(catch_unwind)]
pub fn list_jobs(
&self,
page_token: Option<String>,
page_limit: Option<u32>,
) -> napi::Result<crate::listing::JobListing> {
Ok(crate::listing::JobListing::new(
self.get_inner()?
.list_jobs(crate::listing::options(page_token, page_limit)),
))
}
/// Request cancellation of a server-side job by id. Returns true if the
/// server accepted the cancellation, false if no such job exists.
#[napi(catch_unwind)]
pub async fn cancel_job(&self, job_id: String) -> napi::Result<bool> {
self.get_inner()?.cancel_job(&job_id).await.default_error()
}
/// Pause a server-side job by id: its workers drain and it stays parked
/// until resumed. Returns "pausing", "already_paused", or "committing".
#[napi(catch_unwind)]
pub async fn pause_job(&self, job_id: String) -> napi::Result<String> {
let status = self.get_inner()?.pause_job(&job_id).await.default_error()?;
Ok(match status {
lancedb::database::PauseJobStatus::Pausing => "pausing".to_string(),
lancedb::database::PauseJobStatus::AlreadyPaused => "already_paused".to_string(),
lancedb::database::PauseJobStatus::Committing => "committing".to_string(),
})
}
/// Resume a paused server-side job by id. Returns "resumed",
/// "still_pausing", or "not_paused".
#[napi(catch_unwind)]
pub async fn resume_job(&self, job_id: String) -> napi::Result<String> {
let status = self
.get_inner()?
.resume_job(&job_id)
.await
.default_error()?;
Ok(match status {
lancedb::database::ResumeJobStatus::Resumed => "resumed".to_string(),
lancedb::database::ResumeJobStatus::StillPausing => "still_pausing".to_string(),
lancedb::database::ResumeJobStatus::NotPaused => "not_paused".to_string(),
})
}
#[napi(catch_unwind)]
/// Describe a namespace and return its properties.
pub async fn describe_namespace(
&self,
namespace_path: Vec<String>,
) -> napi::Result<DescribeNamespaceResponse> {
let req = DescribeNamespaceRequest {
id: Some(namespace_path),
..Default::default()
};
let resp = self
.get_inner()?
.describe_namespace(req)
.await
.default_error()?;
Ok(DescribeNamespaceResponse {
properties: resp.properties,
})
}
#[napi(catch_unwind)]
/// List child namespaces under the given namespace path
pub async fn list_namespaces(
&self,
namespace_path: Option<Vec<String>>,
page_token: Option<String>,
limit: Option<u32>,
) -> napi::Result<ListNamespacesResponse> {
let req = ListNamespacesRequest {
id: namespace_path,
page_token,
limit: limit.map(|l| l as i32),
..Default::default()
};
let resp = self
.get_inner()?
.list_namespaces(req)
.await
.default_error()?;
Ok(ListNamespacesResponse {
namespaces: resp.namespaces,
page_token: resp.page_token,
})
}
#[napi(catch_unwind)]
/// Create a new namespace with optional properties.
pub async fn create_namespace(
&self,
namespace_path: Vec<String>,
mode: Option<String>,
properties: Option<HashMap<String, String>>,
) -> napi::Result<CreateNamespaceResponse> {
let mode_str = mode
.map(|m| match m.to_lowercase().as_str() {
"create" => Ok("Create".to_string()),
"exist_ok" => Ok("ExistOk".to_string()),
"overwrite" => Ok("Overwrite".to_string()),
_ => Err(napi::Error::from_reason(format!(
"Invalid mode '{}': expected one of 'create', 'exist_ok', 'overwrite'",
m
))),
})
.transpose()?;
let req = CreateNamespaceRequest {
id: Some(namespace_path),
mode: mode_str,
properties,
..Default::default()
};
let resp = self
.get_inner()?
.create_namespace(req)
.await
.default_error()?;
Ok(CreateNamespaceResponse {
properties: resp.properties,
transaction_id: resp.transaction_id,
})
}
#[napi(catch_unwind)]
/// Drop a namespace.
pub async fn drop_namespace(
&self,
namespace_path: Vec<String>,
mode: Option<String>,
behavior: Option<String>,
) -> napi::Result<DropNamespaceResponse> {
let mode_str = mode
.map(|m| match m.to_lowercase().as_str() {
"skip" => Ok("Skip".to_string()),
"fail" => Ok("Fail".to_string()),
_ => Err(napi::Error::from_reason(format!(
"Invalid mode '{}': expected one of 'skip', 'fail'",
m
))),
})
.transpose()?;
let behavior_str = behavior
.map(|b| match b.to_lowercase().as_str() {
"restrict" => Ok("Restrict".to_string()),
"cascade" => Ok("Cascade".to_string()),
_ => Err(napi::Error::from_reason(format!(
"Invalid behavior '{}': expected one of 'restrict', 'cascade'",
b
))),
})
.transpose()?;
let req = DropNamespaceRequest {
id: Some(namespace_path),
mode: mode_str,
behavior: behavior_str,
..Default::default()
};
let resp = self
.get_inner()?
.drop_namespace(req)
.await
.default_error()?;
Ok(DropNamespaceResponse {
properties: resp.properties,
transaction_id: resp.transaction_id,
})
}
/// Rename a table. `current_namespace_path` and `new_namespace_path` default to
/// the root namespace when omitted; the caller is expected to either pass both
/// or pass neither.
#[napi(catch_unwind)]
pub async fn rename_table(
&self,
current_name: String,
new_name: String,
current_namespace_path: Option<Vec<String>>,
new_namespace_path: Option<Vec<String>>,
) -> napi::Result<()> {
let cur_ns = current_namespace_path.unwrap_or_default();
let new_ns = new_namespace_path.unwrap_or_default();
self.get_inner()?
.drop_all_tables(&namespace)
.rename_table(&current_name, &new_name, &cur_ns, &new_ns)
.await
.default_error()
}
nodejs/src/header.rs
+73
View File
@@ -0,0 +1,73 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use napi::{bindgen_prelude::*, threadsafe_function::ThreadsafeFunction};
use napi_derive::napi;
use std::collections::HashMap;
use std::sync::Arc;
type GetHeadersFn = ThreadsafeFunction<(), Promise<HashMap<String, String>>, (), Status, false>;
/// JavaScript HeaderProvider implementation that wraps a JavaScript callback.
/// This is the only native header provider - all header provider implementations
/// should provide a JavaScript function that returns headers.
#[napi]
pub struct JsHeaderProvider {
get_headers_fn: Arc<GetHeadersFn>,
}
impl Clone for JsHeaderProvider {
fn clone(&self) -> Self {
Self {
get_headers_fn: self.get_headers_fn.clone(),
}
}
}
#[napi]
impl JsHeaderProvider {
/// Create a new JsHeaderProvider from a JavaScript callback
#[napi(constructor)]
pub fn new(
get_headers_callback: Function<(), Promise<HashMap<String, String>>>,
) -> Result<Self> {
let get_headers_fn = get_headers_callback
.build_threadsafe_function()
.build()
.map_err(|e| {
Error::new(
Status::GenericFailure,
format!("Failed to create threadsafe function: {}", e),
)
})?;
Ok(Self {
get_headers_fn: Arc::new(get_headers_fn),
})
}
}
#[cfg(feature = "remote")]
#[async_trait::async_trait]
impl lancedb::remote::HeaderProvider for JsHeaderProvider {
async fn get_headers(&self) -> lancedb::error::Result<HashMap<String, String>> {
// Call the JavaScript function asynchronously
let promise: Promise<HashMap<String, String>> =
self.get_headers_fn.call_async(()).await.map_err(|e| {
lancedb::error::Error::Runtime {
message: format!("Failed to call JavaScript get_headers: {}", e),
}
})?;
// Await the promise result
promise.await.map_err(|e| lancedb::error::Error::Runtime {
message: format!("JavaScript get_headers failed: {}", e),
})
}
}
impl std::fmt::Debug for JsHeaderProvider {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "JsHeaderProvider")
}
}
nodejs/src/index.rs
+122 -7
View File
@@ -3,13 +3,17 @@
use std::sync::Mutex;
use lancedb::index::scalar::{BTreeIndexBuilder, FtsIndexBuilder};
use lancedb::index::Index as LanceDbIndex;
use lancedb::index::scalar::{BTreeIndexBuilder, FmIndexBuilder, FtsIndexBuilder};
use lancedb::index::vector::{
IvfFlatIndexBuilder, IvfHnswPqIndexBuilder, IvfHnswSqIndexBuilder, IvfPqIndexBuilder,
IvfRqIndexBuilder,
};
use lancedb::index::Index as LanceDbIndex;
use lancedb::tokenize as lancedb_tokenize;
use napi_derive::napi;
use crate::error::NapiErrorExt;
use crate::table::FtsToken;
use crate::util::parse_distance_type;
#[napi]
@@ -29,6 +33,67 @@ impl Index {
}
}
#[napi(catch_unwind)]
#[allow(dead_code, clippy::too_many_arguments)]
pub fn tokenize(
query: String,
base_tokenizer: Option<String>,
language: Option<String>,
max_token_length: Option<u32>,
lower_case: Option<bool>,
stem: Option<bool>,
remove_stop_words: Option<bool>,
custom_stop_words: Option<Vec<String>>,
ascii_folding: Option<bool>,
ngram_min_length: Option<u32>,
ngram_max_length: Option<u32>,
prefix_only: Option<bool>,
) -> napi::Result<Vec<FtsToken>> {
let mut opts = FtsIndexBuilder::default();
if let Some(base_tokenizer) = base_tokenizer {
opts = opts.base_tokenizer(base_tokenizer);
}
if let Some(language) = language {
opts = opts.language(&language).map_err(|_| {
napi::Error::from_reason(format!(
"LanceDB does not support the requested language: '{}'",
language
))
})?;
}
if let Some(max_token_length) = max_token_length {
opts = opts.max_token_length(Some(max_token_length as usize));
}
if let Some(lower_case) = lower_case {
opts = opts.lower_case(lower_case);
}
if let Some(stem) = stem {
opts = opts.stem(stem);
}
if let Some(remove_stop_words) = remove_stop_words {
opts = opts.remove_stop_words(remove_stop_words);
}
opts = opts.custom_stop_words(custom_stop_words);
if let Some(ascii_folding) = ascii_folding {
opts = opts.ascii_folding(ascii_folding);
}
if let Some(ngram_min_length) = ngram_min_length {
opts = opts.ngram_min_length(ngram_min_length);
}
if let Some(ngram_max_length) = ngram_max_length {
opts = opts.ngram_max_length(ngram_max_length);
}
if let Some(prefix_only) = prefix_only {
opts = opts.ngram_prefix_only(prefix_only);
}
Ok(lancedb_tokenize(&query, &opts)
.default_error()?
.into_iter()
.map(FtsToken::from)
.collect())
}
#[napi]
impl Index {
#[napi(factory)]
@@ -65,6 +130,36 @@ impl Index {
})
}
#[napi(factory)]
pub fn ivf_rq(
distance_type: Option<String>,
num_partitions: Option<u32>,
num_bits: Option<u32>,
max_iterations: Option<u32>,
sample_rate: Option<u32>,
) -> napi::Result<Self> {
let mut ivf_rq_builder = IvfRqIndexBuilder::default();
if let Some(distance_type) = distance_type {
let distance_type = parse_distance_type(distance_type)?;
ivf_rq_builder = ivf_rq_builder.distance_type(distance_type);
}
if let Some(num_partitions) = num_partitions {
ivf_rq_builder = ivf_rq_builder.num_partitions(num_partitions);
}
if let Some(num_bits) = num_bits {
ivf_rq_builder = ivf_rq_builder.num_bits(num_bits);
}
if let Some(max_iterations) = max_iterations {
ivf_rq_builder = ivf_rq_builder.max_iterations(max_iterations);
}
if let Some(sample_rate) = sample_rate {
ivf_rq_builder = ivf_rq_builder.sample_rate(sample_rate);
}
Ok(Self {
inner: Mutex::new(Some(LanceDbIndex::IvfRq(ivf_rq_builder))),
})
}
#[napi(factory)]
pub fn ivf_flat(
distance_type: Option<String>,
@@ -112,6 +207,13 @@ impl Index {
}
}
#[napi(factory)]
pub fn fm() -> Self {
Self {
inner: Mutex::new(Some(LanceDbIndex::Fm(FmIndexBuilder::default()))),
}
}
#[napi(factory)]
#[allow(clippy::too_many_arguments)]
pub fn fts(
@@ -122,11 +224,13 @@ impl Index {
lower_case: Option<bool>,
stem: Option<bool>,
remove_stop_words: Option<bool>,
custom_stop_words: Option<Vec<String>>,
ascii_folding: Option<bool>,
ngram_min_length: Option<u32>,
ngram_max_length: Option<u32>,
prefix_only: Option<bool>,
) -> Self {
block_size: Option<u32>,
) -> napi::Result<Self> {
let mut opts = FtsIndexBuilder::default();
if let Some(with_position) = with_position {
opts = opts.with_position(with_position);
@@ -135,7 +239,12 @@ impl Index {
opts = opts.base_tokenizer(base_tokenizer);
}
if let Some(language) = language {
opts = opts.language(&language).unwrap();
opts = opts.language(&language).map_err(|_| {
napi::Error::from_reason(format!(
"LanceDB does not support the requested language: '{}'",
language
))
})?;
}
if let Some(max_token_length) = max_token_length {
opts = opts.max_token_length(Some(max_token_length as usize));
@@ -149,6 +258,7 @@ impl Index {
if let Some(remove_stop_words) = remove_stop_words {
opts = opts.remove_stop_words(remove_stop_words);
}
opts = opts.custom_stop_words(custom_stop_words);
if let Some(ascii_folding) = ascii_folding {
opts = opts.ascii_folding(ascii_folding);
}
@@ -161,10 +271,15 @@ impl Index {
if let Some(prefix_only) = prefix_only {
opts = opts.ngram_prefix_only(prefix_only);
}
Self {
inner: Mutex::new(Some(LanceDbIndex::FTS(opts))),
if let Some(block_size) = block_size {
opts = opts
.block_size(block_size as usize)
.map_err(|err| napi::Error::from_reason(err.to_string()))?;
}
Ok(Self {
inner: Mutex::new(Some(LanceDbIndex::FTS(opts))),
})
}
#[napi(factory)]
nodejs/src/iterator.rs
+8
View File
@@ -7,6 +7,8 @@ use lancedb::ipc::batches_to_ipc_file;
use napi::bindgen_prelude::*;
use napi_derive::napi;
use crate::util::schema_to_buffer;
/** Typescript-style Async Iterator over RecordBatches */
#[napi]
pub struct RecordBatchIterator {
@@ -19,6 +21,12 @@ impl RecordBatchIterator {
Self { inner }
}
/// Return the executed stream's output schema as an Arrow IPC file.
#[napi]
pub fn schema(&self) -> napi::Result<Buffer> {
schema_to_buffer(&self.inner.schema())
}
#[napi(catch_unwind)]
pub async unsafe fn next(&mut self) -> napi::Result<Option<Buffer>> {
if let Some(rst) = self.inner.next().await {
nodejs/src/job.rs
+182
View File
@@ -0,0 +1,182 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::sync::Arc;
use arrow_array::RecordBatch;
use lancedb::job::JobEventsRequest;
use napi::bindgen_prelude::Buffer;
use napi_derive::napi;
use crate::error::NapiErrorExt;
/// A handle to an operation that may still be running.
#[napi]
pub struct Job {
inner: Arc<lancedb::Job>,
}
impl Job {
pub(crate) fn new<T>(inner: lancedb::Job<T>) -> Self
where
T: Clone + Send + Sync + 'static,
{
Self {
inner: Arc::new(inner.map(|_| ())),
}
}
}
#[napi]
impl Job {
/// Identifies the operation on the server that is running it. Operations
/// that run in this process have no server id. The value is opaque.
#[napi(getter)]
pub fn id(&self) -> Option<String> {
self.inner.id().map(str::to_string)
}
/// The operation's current lifecycle state: "running", "finished",
/// "failed", or "cancelled".
///
/// A point snapshot; unlike {@link Job.wait} it does not block or reject
/// on a terminal failure state. States a newer server reports that this
/// client version does not know pass through as-is.
#[napi(catch_unwind)]
pub async fn status(&self) -> napi::Result<String> {
self.inner.status().await.default_error()
}
/// Wait until the operation reaches a terminal state.
#[napi(catch_unwind)]
pub async fn wait(&self) -> napi::Result<()> {
self.inner.wait().await.default_error()
}
/// Request cancellation. Cancelling a finished operation is a no-op.
#[napi(catch_unwind)]
pub async fn cancel(&self) -> napi::Result<()> {
self.inner.cancel().await.default_error()
}
/// Ask the backend for this job's current state, and for a server-side job
/// its full record, then cache it for the getters below.
///
/// They are all null until this runs, because submitting an operation
/// returns only a job id. {@link Job.status} fetches the whole record too;
/// {@link Job.wait} records only the terminal state it establishes.
#[napi(catch_unwind)]
pub async fn refresh(&self) -> napi::Result<()> {
self.inner.refresh().await.default_error()
}
/// The last observed lifecycle state, without contacting the backend.
#[napi(getter)]
pub fn state(&self) -> Option<String> {
self.inner.state()
}
/// The job's type, as the server names it. Null for an in-process job,
/// which has no server-side record.
#[napi(getter)]
pub fn job_type(&self) -> Option<String> {
self.inner.job_type()
}
/// When the job was created, in milliseconds since the epoch.
#[napi(getter)]
pub fn creation_ms(&self) -> Option<i64> {
self.inner.creation_ms()
}
/// The job-type-specific specification as a JSON string, when present.
#[napi(getter)]
pub fn spec_json(&self) -> Option<String> {
self.inner.spec().map(|spec| spec.to_string())
}
/// The job-type-specific terminal result as a JSON string. Null until the
/// job succeeds, so a job that never terminates reports its progress
/// through {@link Job.events} instead.
#[napi(getter)]
pub fn result_json(&self) -> Option<String> {
self.inner.result().map(|result| result.to_string())
}
/// Why the job failed, when it failed and the server reports a reason.
#[napi(getter)]
pub fn failure(&self) -> Option<JobFailureInfo> {
self.inner.failure().map(|failure| JobFailureInfo {
phase: failure.phase,
message: failure.message,
retryable: failure.retryable,
})
}
/// This job's recorded lifecycle events, as an Arrow IPC stream buffer.
/// The TypeScript wrapper turns it into an Arrow table.
#[napi(catch_unwind)]
pub async fn events(&self, limit: Option<u32>, filter: Option<String>) -> napi::Result<Buffer> {
let batches = self
.inner
.events(JobEventsRequest { limit, filter })
.await
.default_error()?;
batches_to_ipc_buffer(&batches)
}
}
/// Serialise Arrow batches as a single IPC stream for the TypeScript layer.
fn batches_to_ipc_buffer(batches: &[RecordBatch]) -> napi::Result<Buffer> {
let Some(first) = batches.first() else {
return Ok(Buffer::from(Vec::<u8>::new()));
};
let mut out = Vec::new();
let mut writer = arrow_ipc::writer::StreamWriter::try_new(&mut out, &first.schema())
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
for batch in batches {
writer
.write(batch)
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
}
writer
.finish()
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
drop(writer);
Ok(Buffer::from(out))
}
/// A row from `Connection.listJobs`: one server-side job.
#[napi(object)]
pub struct JobInfo {
/// The job id -- what `Connection.openJob` and `Connection.cancelJob`
/// accept.
pub job_id: String,
/// The table the job runs against, without URI or namespace.
pub table: String,
pub job_type: String,
/// Lifecycle state: "running", "finished", "failed", or "cancelled".
pub state: String,
/// When the job was created, in milliseconds since the epoch.
pub created_at_millis: i64,
}
impl From<lancedb::database::JobInfo> for JobInfo {
fn from(info: lancedb::database::JobInfo) -> Self {
Self {
job_id: info.job_id,
table: info.table,
job_type: info.job_type,
state: info.state,
created_at_millis: info.created_at_millis,
}
}
}
/// The server's account of why a job failed.
#[napi(object)]
pub struct JobFailureInfo {
pub phase: Option<String>,
pub message: Option<String>,
pub retryable: Option<bool>,
}
nodejs/src/lib.rs
+58 -10
View File
@@ -1,19 +1,31 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// The materialized-view refresh future deepens the type graph past the
// default trait-recursion depth; same raise as the core crate applies.
#![recursion_limit = "256"]
use std::collections::HashMap;
use env_logger::Env;
use napi_derive::*;
mod blob;
mod catalog;
mod connection;
mod error;
mod header;
mod index;
mod iterator;
mod job;
mod listing;
pub mod merge;
pub mod otel;
pub mod permutation;
mod query;
pub mod remote;
mod rerankers;
mod scannable;
mod session;
mod table;
mod util;
@@ -21,20 +33,31 @@ mod util;
#[napi(object)]
#[derive(Debug)]
pub struct ConnectionOptions {
/// (For LanceDB OSS only): The interval, in seconds, at which to check for
/// updates to the table from other processes. If None, then consistency is not
/// checked. For performance reasons, this is the default. For strong
/// consistency, set this to zero seconds. Then every read will check for
/// updates from other processes. As a compromise, you can set this to a
/// non-zero value for eventual consistency. If more than that interval
/// has passed since the last check, then the table will be checked for updates.
/// Note: this consistency only applies to read operations. Write operations are
/// The interval, in seconds, at which to check for updates to the table
/// from other processes. If None, then consistency is not checked. For
/// performance reasons, this is the default. For strong consistency, set
/// this to zero seconds. Then every read will check for updates from other
/// processes. As a compromise, you can set this to a non-zero value for
/// eventual consistency. If more than that interval has passed since the
/// last check, then the table will be checked for updates. Note: this
/// consistency only applies to read operations. Write operations are
/// always consistent.
///
/// Stronger consistency is not free. The smaller the interval, the more
/// often each read pays the cost of checking for updates against object
/// storage, raising per-read latency and cost.
pub read_consistency_interval: Option<f64>,
/// (For LanceDB OSS only): configuration for object storage.
///
/// The available options are described at https://lancedb.github.io/lancedb/guides/storage/
/// The available options are described at https://docs.lancedb.com/storage/
pub storage_options: Option<HashMap<String, String>>,
/// (For LanceDB OSS only): use directory namespace manifests as the source
/// of truth for table metadata. Existing directory-listed root tables are
/// migrated into the manifest on access.
pub manifest_enabled: Option<bool>,
/// (For LanceDB OSS only): extra properties for the backing namespace
/// client used by manifest-enabled native connections.
pub namespace_client_properties: Option<HashMap<String, String>>,
/// (For LanceDB OSS only): the session to use for this connection. Holds
/// shared caches and other session-specific state.
pub session: Option<session::Session>,
@@ -51,6 +74,11 @@ pub struct ConnectionOptions {
/// (For LanceDB cloud only): the host to use for LanceDB cloud. Used
/// for testing purposes.
pub host_override: Option<String>,
/// (For LanceDB cloud only): OAuth configuration for IdP-based
/// authentication (e.g., Azure Entra ID). When set, token acquisition
/// and refresh are handled entirely in Rust. TypeScript users should pass
/// the public `OAuthConfig` type exported from `@lancedb/lancedb`.
pub oauth_config: Option<remote::OAuthConfig>,
}
#[napi(object)]
@@ -58,7 +86,27 @@ pub struct OpenTableOptions {
pub storage_options: Option<HashMap<String, String>>,
}
#[napi::module_init]
#[napi(object)]
#[derive(Debug)]
pub struct ConnectNamespaceOptions {
/// The interval, in seconds, at which to check for updates to the table
/// from other processes. If None, then consistency is not checked. For
/// performance reasons, this is the default. For strong consistency, set
/// this to zero seconds. Then every read will check for updates from other
/// processes. As a compromise, you can set this to a non-zero value for
/// eventual consistency.
pub read_consistency_interval: Option<f64>,
/// Configuration for object storage. The available options are described
/// at https://docs.lancedb.com/storage/
pub storage_options: Option<HashMap<String, String>>,
/// Extra properties for the backing namespace client.
pub namespace_client_properties: Option<HashMap<String, String>>,
/// The session to use for this connection. Holds shared caches and other
/// session-specific state.
pub session: Option<session::Session>,
}
#[napi_derive::module_init]
fn init() {
let env = Env::new()
.filter_or("LANCEDB_LOG", "warn")
nodejs/src/listing.rs
+83
View File
@@ -0,0 +1,83 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use crate::error::NapiErrorExt;
use futures::{StreamExt, future::poll_fn};
use lancedb::listing::{Listing, ListingOptions};
use napi_derive::napi;
use std::sync::Mutex;
pub fn options(page_token: Option<String>, page_limit: Option<u32>) -> ListingOptions {
let mut options = ListingOptions::default();
options.page_token = page_token;
options.page_limit = page_limit;
options
}
#[napi]
pub struct NameListing {
inner: Mutex<Listing<String>>,
next_lock: futures::lock::Mutex<()>,
}
impl NameListing {
pub fn new(inner: Listing<String>) -> Self {
Self {
inner: Mutex::new(inner),
next_lock: futures::lock::Mutex::new(()),
}
}
}
#[napi]
impl NameListing {
#[napi]
pub fn num_page_results(&self) -> u32 {
self.inner.lock().unwrap().num_page_results() as u32
}
#[napi]
pub fn page_token(&self) -> Option<String> {
self.inner.lock().unwrap().page_token().map(str::to_owned)
}
#[napi]
pub async fn next(&self) -> napi::Result<Option<String>> {
let _guard = self.next_lock.lock().await;
let value = poll_fn(|cx| self.inner.lock().unwrap().poll_next_unpin(cx))
.await
.transpose()
.default_error()?;
Ok(value)
}
}
#[napi]
pub struct JobListing {
inner: Mutex<Listing<lancedb::database::JobInfo>>,
next_lock: futures::lock::Mutex<()>,
}
impl JobListing {
pub fn new(inner: Listing<lancedb::database::JobInfo>) -> Self {
Self {
inner: Mutex::new(inner),
next_lock: futures::lock::Mutex::new(()),
}
}
}
#[napi]
impl JobListing {
#[napi]
pub fn num_page_results(&self) -> u32 {
self.inner.lock().unwrap().num_page_results() as u32
}
#[napi]
pub fn page_token(&self) -> Option<String> {
self.inner.lock().unwrap().page_token().map(str::to_owned)
}
#[napi]
pub async fn next(&self) -> napi::Result<Option<crate::job::JobInfo>> {
let _guard = self.next_lock.lock().await;
let value = poll_fn(|cx| self.inner.lock().unwrap().poll_next_unpin(cx))
.await
.transpose()
.default_error()?;
Ok(value.map(|value| value.into()))
}
}
nodejs/src/merge.rs
+25 -6
View File
@@ -3,7 +3,7 @@
use std::time::Duration;
use lancedb::{arrow::IntoArrow, ipc::ipc_file_to_batches, table::merge::MergeInsertBuilder};
use lancedb::{ipc::ipc_file_to_batches, table::merge::MergeInsertBuilder};
use napi::bindgen_prelude::*;
use napi_derive::napi;
@@ -43,13 +43,32 @@ impl NativeMergeInsertBuilder {
self.inner.timeout(Duration::from_millis(timeout as u64));
}
#[napi]
pub fn use_index(&self, use_index: bool) -> Self {
let mut this = self.clone();
this.inner.use_index(use_index);
this
}
#[napi(js_name = "useRTE")]
pub fn use_rte(&self, enable: bool) -> Self {
let mut this = self.clone();
this.inner.use_rte(enable);
this
}
#[napi]
pub fn validate_single_shard(&self, validate_single_shard: bool) -> Self {
let mut this = self.clone();
this.inner.validate_single_shard(validate_single_shard);
this
}
#[napi(catch_unwind)]
pub async fn execute(&self, buf: Buffer) -> napi::Result<MergeResult> {
let data = ipc_file_to_batches(buf.to_vec())
.and_then(IntoArrow::into_arrow)
.map_err(|e| {
napi::Error::from_reason(format!("Failed to read IPC file: {}", convert_error(&e)))
})?;
let data = ipc_file_to_batches(buf.to_vec()).map_err(|e| {
napi::Error::from_reason(format!("Failed to read IPC file: {}", convert_error(&e)))
})?;
let this = self.clone();
nodejs/src/otel.rs
+119
View File
@@ -0,0 +1,119 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
//! Node.js bindings over [`lancedb::metrics_otel`].
//!
//! The aggregation, catalog, and histogram bucketing all live in the LanceDB
//! core crate; this module only converts the core snapshot types into napi
//! objects and exposes the three entry points to JavaScript, where
//! `lancedb/otel.ts` bridges them into the user's OpenTelemetry `MeterProvider`.
use std::collections::HashMap;
use lancedb::metrics_otel::{MetricPoint as CoreMetricPoint, MetricValue};
use napi_derive::napi;
/// One cumulative histogram bucket: all samples with value `<= le`.
#[napi(object)]
pub struct MetricBucket {
/// The inclusive upper bound of the bucket, or `"+Inf"` for the final bucket.
pub le: String,
/// Cumulative number of samples less than or equal to `le`.
pub cumulative_count: f64,
}
/// One aggregated metric data point. For counters and gauges only `value` is
/// set; for histograms `buckets` (cumulative `le` counts), `count`, and `sum`
/// are set.
#[napi(object)]
pub struct MetricPoint {
pub name: String,
pub kind: String,
pub attributes: HashMap<String, String>,
pub value: Option<f64>,
pub buckets: Option<Vec<MetricBucket>>,
pub count: Option<f64>,
pub sum: Option<f64>,
}
impl From<CoreMetricPoint> for MetricPoint {
fn from(point: CoreMetricPoint) -> Self {
let kind = point.kind.as_str().to_string();
let (value, buckets, count, sum) = match point.value {
MetricValue::Scalar(v) => (Some(v), None, None, None),
MetricValue::Histogram {
buckets,
count,
sum,
} => (
None,
Some(
buckets
.into_iter()
// Counts stay well within the f64-exact integer range
// (2^53), so this cast is lossless in practice and keeps
// the values plain JS numbers for OpenTelemetry.
.map(|(le, cumulative_count)| MetricBucket {
le,
cumulative_count: cumulative_count as f64,
})
.collect(),
),
Some(count as f64),
Some(sum),
),
};
Self {
name: point.name,
kind,
attributes: point.attributes,
value,
buckets,
count,
sum,
}
}
}
/// A described metric, used by the JavaScript layer to create instruments up front.
#[napi(object)]
pub struct MetricDescription {
pub name: String,
pub kind: String,
pub unit: Option<String>,
pub description: String,
}
/// Install the LanceDB metrics recorder as the process-global `metrics` recorder.
///
/// Returns `true` if the recorder is installed (now or previously). Returns
/// `false` if a *different* recorder is already installed — `metrics` allows
/// only one global recorder per process, so LanceDB cannot coexist with another.
#[napi]
pub fn register_lancedb_metrics_recorder() -> bool {
lancedb::metrics_otel::register_metrics_recorder()
}
/// The catalog of described LanceDB metrics. Empty until the recorder is installed.
#[napi]
pub fn lancedb_metrics_catalog() -> Vec<MetricDescription> {
lancedb::metrics_otel::metrics_catalog()
.into_iter()
.map(|desc| MetricDescription {
name: desc.name,
kind: desc.kind.as_str().to_string(),
unit: desc.unit,
description: desc.description,
})
.collect()
}
/// A point-in-time snapshot of every recorded metric. Empty until the recorder
/// is installed.
#[napi]
pub fn snapshot_lancedb_metrics() -> Vec<MetricPoint> {
lancedb::metrics_otel::snapshot_metrics()
.into_iter()
.map(MetricPoint::from)
.collect()
}
nodejs/src/permutation.rs
+254
View File
@@ -0,0 +1,254 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::sync::{Arc, Mutex};
use crate::{error::NapiErrorExt, table::Table};
use lancedb::dataloader::{
permutation::builder::{PermutationBuilder as LancePermutationBuilder, ShuffleStrategy},
permutation::split::{SplitSizes, SplitStrategy},
};
use napi_derive::napi;
#[napi(object)]
pub struct SplitRandomOptions {
pub ratios: Option<Vec<f64>>,
pub counts: Option<Vec<i64>>,
pub fixed: Option<i64>,
pub seed: Option<i64>,
pub clump_size: Option<i64>,
pub split_names: Option<Vec<String>>,
}
#[napi(object)]
pub struct SplitHashOptions {
pub columns: Vec<String>,
pub split_weights: Vec<i64>,
pub discard_weight: Option<i64>,
pub split_names: Option<Vec<String>>,
}
#[napi(object)]
pub struct SplitSequentialOptions {
pub ratios: Option<Vec<f64>>,
pub counts: Option<Vec<i64>>,
pub fixed: Option<i64>,
pub split_names: Option<Vec<String>>,
}
#[napi(object)]
pub struct SplitCalculatedOptions {
pub calculation: String,
pub split_names: Option<Vec<String>>,
}
#[napi(object)]
pub struct ShuffleOptions {
pub seed: Option<i64>,
pub clump_size: Option<i64>,
}
pub struct PermutationBuilderState {
pub builder: Option<LancePermutationBuilder>,
}
#[napi]
pub struct PermutationBuilder {
state: Arc<Mutex<PermutationBuilderState>>,
}
impl PermutationBuilder {
pub fn new(builder: LancePermutationBuilder) -> Self {
Self {
state: Arc::new(Mutex::new(PermutationBuilderState {
builder: Some(builder),
})),
}
}
}
impl PermutationBuilder {
fn modify(
&self,
func: impl FnOnce(LancePermutationBuilder) -> LancePermutationBuilder,
) -> napi::Result<Self> {
let mut state = self.state.lock().unwrap();
let builder = state
.builder
.take()
.ok_or_else(|| napi::Error::from_reason("Builder already consumed"))?;
state.builder = Some(func(builder));
Ok(Self {
state: self.state.clone(),
})
}
}
#[napi]
impl PermutationBuilder {
#[napi]
pub fn persist(
&self,
connection: &crate::connection::Connection,
table_name: String,
) -> napi::Result<Self> {
let database = connection.database()?;
self.modify(|builder| builder.persist(database, table_name))
}
/// Configure random splits
#[napi]
pub fn split_random(&self, options: SplitRandomOptions) -> napi::Result<Self> {
// Check that exactly one split type is provided
let split_args_count = [
options.ratios.is_some(),
options.counts.is_some(),
options.fixed.is_some(),
]
.iter()
.filter(|&&x| x)
.count();
if split_args_count != 1 {
return Err(napi::Error::from_reason(
"Exactly one of 'ratios', 'counts', or 'fixed' must be provided",
));
}
let sizes = if let Some(ratios) = options.ratios {
SplitSizes::Percentages(ratios)
} else if let Some(counts) = options.counts {
SplitSizes::Counts(counts.into_iter().map(|c| c as u64).collect())
} else if let Some(fixed) = options.fixed {
SplitSizes::Fixed(fixed as u64)
} else {
unreachable!("One of the split arguments must be provided");
};
let seed = options.seed.map(|s| s as u64);
let clump_size = options.clump_size.map(|c| c as u64);
self.modify(|builder| {
builder.with_split_strategy(
SplitStrategy::Random {
seed,
sizes,
clump_size,
},
options.split_names.clone(),
)
})
}
/// Configure hash-based splits
#[napi]
pub fn split_hash(&self, options: SplitHashOptions) -> napi::Result<Self> {
let split_weights = options
.split_weights
.into_iter()
.map(|w| w as u64)
.collect();
let discard_weight = options.discard_weight.unwrap_or(0) as u64;
self.modify(move |builder| {
builder.with_split_strategy(
SplitStrategy::Hash {
columns: options.columns,
split_weights,
discard_weight,
},
options.split_names,
)
})
}
/// Configure sequential splits
#[napi]
pub fn split_sequential(&self, options: SplitSequentialOptions) -> napi::Result<Self> {
// Check that exactly one split type is provided
let split_args_count = [
options.ratios.is_some(),
options.counts.is_some(),
options.fixed.is_some(),
]
.iter()
.filter(|&&x| x)
.count();
if split_args_count != 1 {
return Err(napi::Error::from_reason(
"Exactly one of 'ratios', 'counts', or 'fixed' must be provided",
));
}
let sizes = if let Some(ratios) = options.ratios {
SplitSizes::Percentages(ratios)
} else if let Some(counts) = options.counts {
SplitSizes::Counts(counts.into_iter().map(|c| c as u64).collect())
} else if let Some(fixed) = options.fixed {
SplitSizes::Fixed(fixed as u64)
} else {
unreachable!("One of the split arguments must be provided");
};
self.modify(move |builder| {
builder.with_split_strategy(SplitStrategy::Sequential { sizes }, options.split_names)
})
}
/// Configure calculated splits
#[napi]
pub fn split_calculated(&self, options: SplitCalculatedOptions) -> napi::Result<Self> {
self.modify(move |builder| {
builder.with_split_strategy(
SplitStrategy::Calculated {
calculation: options.calculation,
},
options.split_names,
)
})
}
/// Configure shuffling
#[napi]
pub fn shuffle(&self, options: ShuffleOptions) -> napi::Result<Self> {
let seed = options.seed.map(|s| s as u64);
let clump_size = options.clump_size.map(|c| c as u64);
self.modify(|builder| {
builder.with_shuffle_strategy(ShuffleStrategy::Random { seed, clump_size })
})
}
/// Configure filtering
#[napi]
pub fn filter(&self, filter: String) -> napi::Result<Self> {
self.modify(|builder| builder.with_filter(filter))
}
/// Execute the permutation builder and create the table
#[napi]
pub async fn execute(&self) -> napi::Result<Table> {
let builder = {
let mut state = self.state.lock().unwrap();
state
.builder
.take()
.ok_or_else(|| napi::Error::from_reason("Builder already consumed"))?
};
let table = builder.build().await.default_error()?;
Ok(Table::new(table))
}
}
/// Create a permutation builder for the given table
#[napi]
pub fn permutation_builder(table: &crate::table::Table) -> napi::Result<PermutationBuilder> {
use lancedb::dataloader::permutation::builder::PermutationBuilder as LancePermutationBuilder;
let inner_table = table.inner_ref()?.clone();
let inner_builder = LancePermutationBuilder::new(inner_table);
Ok(PermutationBuilder::new(inner_builder))
}
nodejs/src/query.rs
+247 -56
View File
@@ -3,26 +3,99 @@
use std::sync::Arc;
use crate::error::NapiErrorExt;
use crate::error::convert_error;
use crate::iterator::RecordBatchIterator;
use crate::rerankers::RerankHybridCallbackArgs;
use crate::rerankers::Reranker;
use crate::util::{parse_distance_type, schema_to_buffer};
use arrow_array::{
Array, Float16Array as ArrowFloat16Array, Float32Array as ArrowFloat32Array,
Float64Array as ArrowFloat64Array, UInt8Array as ArrowUInt8Array,
};
use arrow_buffer::ScalarBuffer;
use half::f16;
use lancedb::index::scalar::{
BooleanQuery, BoostQuery, FtsQuery, FullTextSearchQuery, MatchQuery, MultiMatchQuery, Occur,
Operator, PhraseQuery,
};
use lancedb::query::AnalyzePlanDistributedMetrics;
use lancedb::query::ExecutableQuery;
use lancedb::query::Query as LanceDbQuery;
use lancedb::query::QueryBase;
use lancedb::query::QueryExecutionOptions;
use lancedb::query::Select;
use lancedb::query::TakeQuery as LanceDbTakeQuery;
use lancedb::query::VectorQuery as LanceDbVectorQuery;
use lancedb::query::{ColumnOrdering as LanceDbColumnOrdering, VectorQuery as LanceDbVectorQuery};
use napi::bindgen_prelude::*;
use napi_derive::napi;
use crate::error::convert_error;
use crate::error::NapiErrorExt;
use crate::iterator::RecordBatchIterator;
use crate::rerankers::Reranker;
use crate::rerankers::RerankerCallbacks;
use crate::util::parse_distance_type;
#[napi(object)]
pub struct ColumnOrdering {
pub ascending: bool,
pub nulls_first: bool,
pub column_name: String,
}
impl From<ColumnOrdering> for LanceDbColumnOrdering {
fn from(value: ColumnOrdering) -> Self {
match (value.ascending, value.nulls_first) {
(true, true) => Self::asc_nulls_first(value.column_name),
(true, false) => Self::asc_nulls_last(value.column_name),
(false, true) => Self::desc_nulls_first(value.column_name),
(false, false) => Self::desc_nulls_last(value.column_name),
}
}
}
fn analyze_plan_options(
distributed_metrics: Option<String>,
) -> napi::Result<QueryExecutionOptions> {
let analyze_plan_distributed_metrics =
match distributed_metrics.as_deref().unwrap_or("aggregate") {
"aggregate" => AnalyzePlanDistributedMetrics::Aggregate,
"per_worker" => AnalyzePlanDistributedMetrics::PerWorker,
"full" => AnalyzePlanDistributedMetrics::Full,
mode => {
return Err(napi::Error::from_reason(format!(
"Invalid distributedMetrics value '{}'. Expected one of: \
'aggregate', 'per_worker', 'full'",
mode
)));
}
};
let mut options = QueryExecutionOptions::default();
options.analyze_plan_distributed_metrics = analyze_plan_distributed_metrics;
Ok(options)
}
fn bytes_to_arrow_array(data: Uint8Array, dtype: String) -> napi::Result<Arc<dyn Array>> {
let buf = arrow_buffer::Buffer::from(data.to_vec());
let num_bytes = buf.len();
match dtype.as_str() {
"float16" => {
let scalar_buf = ScalarBuffer::<f16>::new(buf, 0, num_bytes / 2);
Ok(Arc::new(ArrowFloat16Array::new(scalar_buf, None)))
}
"float32" => {
let scalar_buf = ScalarBuffer::<f32>::new(buf, 0, num_bytes / 4);
Ok(Arc::new(ArrowFloat32Array::new(scalar_buf, None)))
}
"float64" => {
let scalar_buf = ScalarBuffer::<f64>::new(buf, 0, num_bytes / 8);
Ok(Arc::new(ArrowFloat64Array::new(scalar_buf, None)))
}
"uint8" => {
let scalar_buf = ScalarBuffer::<u8>::new(buf, 0, num_bytes);
Ok(Arc::new(ArrowUInt8Array::new(scalar_buf, None)))
}
_ => Err(napi::Error::from_reason(format!(
"Unsupported vector dtype: {}. Expected one of: float16, float32, float64, uint8",
dtype
))),
}
}
#[napi]
pub struct Query {
@@ -42,7 +115,7 @@ impl Query {
}
#[napi]
pub fn full_text_search(&mut self, query: napi::JsObject) -> napi::Result<()> {
pub fn full_text_search(&mut self, query: Object) -> napi::Result<()> {
let query = parse_fts_query(query)?;
self.inner = self.inner.clone().full_text_search(query);
Ok(())
@@ -78,6 +151,13 @@ impl Query {
Ok(VectorQuery { inner })
}
#[napi]
pub fn nearest_to_raw(&mut self, data: Uint8Array, dtype: String) -> Result<VectorQuery> {
let array = bytes_to_arrow_array(data, dtype)?;
let inner = self.inner.clone().nearest_to(array).default_error()?;
Ok(VectorQuery { inner })
}
#[napi]
pub fn fast_search(&mut self) {
self.inner = self.inner.clone().fast_search();
@@ -93,11 +173,35 @@ impl Query {
self.inner = self.inner.clone().disable_scoring_autoprojection();
}
#[napi(js_name = "useRTE")]
pub fn use_rte(&mut self, enable: bool) {
self.inner = self.inner.clone().use_rte(enable);
}
#[napi]
pub fn order_by(&mut self, ordering: Option<Vec<ColumnOrdering>>) -> napi::Result<()> {
let ordering = ordering.map(|ordering| {
ordering
.into_iter()
.map(LanceDbColumnOrdering::from)
.collect()
});
self.inner = self.inner.clone().order_by(ordering);
Ok(())
}
#[napi(catch_unwind)]
pub async fn output_schema(&self) -> napi::Result<Buffer> {
let schema = self.inner.output_schema().await.default_error()?;
schema_to_buffer(&schema)
}
#[napi(catch_unwind)]
pub async fn execute(
&self,
max_batch_length: Option<u32>,
timeout_ms: Option<u32>,
with_row_id: Option<bool>,
) -> napi::Result<RecordBatchIterator> {
let mut execution_opts = QueryExecutionOptions::default();
if let Some(max_batch_length) = max_batch_length {
@@ -106,8 +210,12 @@ impl Query {
if let Some(timeout_ms) = timeout_ms {
execution_opts.timeout = Some(std::time::Duration::from_millis(timeout_ms as u64))
}
let inner_stream = self
.inner
let query = if with_row_id == Some(true) {
self.inner.clone().with_row_id()
} else {
self.inner.clone()
};
let inner_stream = query
.execute_with_options(execution_opts)
.await
.map_err(|e| {
@@ -130,13 +238,17 @@ impl Query {
}
#[napi(catch_unwind)]
pub async fn analyze_plan(&self) -> napi::Result<String> {
self.inner.analyze_plan().await.map_err(|e| {
napi::Error::from_reason(format!(
"Failed to execute analyze plan: {}",
convert_error(&e)
))
})
pub async fn analyze_plan(&self, distributed_metrics: Option<String>) -> napi::Result<String> {
let options = analyze_plan_options(distributed_metrics)?;
self.inner
.analyze_plan_with_options(options)
.await
.map_err(|e| {
napi::Error::from_reason(format!(
"Failed to execute analyze plan: {}",
convert_error(&e)
))
})
}
}
@@ -162,6 +274,13 @@ impl VectorQuery {
Ok(())
}
#[napi]
pub fn add_query_vector_raw(&mut self, data: Uint8Array, dtype: String) -> Result<()> {
let array = bytes_to_arrow_array(data, dtype)?;
self.inner = self.inner.clone().add_query_vector(array).default_error()?;
Ok(())
}
#[napi]
pub fn distance_type(&mut self, distance_type: String) -> napi::Result<()> {
let distance_type = parse_distance_type(distance_type)?;
@@ -180,8 +299,23 @@ impl VectorQuery {
}
#[napi]
pub fn nprobes(&mut self, nprobe: u32) {
self.inner = self.inner.clone().nprobes(nprobe as usize);
pub fn nprobes(&mut self, nprobe: u32) -> napi::Result<()> {
self.inner = self
.inner
.clone()
.nprobes(nprobe as usize)
.default_error()?;
Ok(())
}
#[napi]
pub fn search_effort(&mut self, search_effort: Option<f64>) -> napi::Result<()> {
self.inner = self
.inner
.clone()
.search_effort(search_effort)
.default_error()?;
Ok(())
}
#[napi]
@@ -234,7 +368,7 @@ impl VectorQuery {
}
#[napi]
pub fn full_text_search(&mut self, query: napi::JsObject) -> napi::Result<()> {
pub fn full_text_search(&mut self, query: Object) -> napi::Result<()> {
let query = parse_fts_query(query)?;
self.inner = self.inner.clone().full_text_search(query);
Ok(())
@@ -270,17 +404,42 @@ impl VectorQuery {
self.inner = self.inner.clone().with_row_id();
}
#[napi(js_name = "useRTE")]
pub fn use_rte(&mut self, enable: bool) {
self.inner = self.inner.clone().use_rte(enable);
}
#[napi]
pub fn disable_scoring_autoprojection(&mut self) {
self.inner = self.inner.clone().disable_scoring_autoprojection();
}
#[napi]
pub fn rerank(&mut self, callbacks: RerankerCallbacks) {
self.inner = self
.inner
.clone()
.rerank(Arc::new(Reranker::new(callbacks)));
pub fn rerank(
&mut self,
rerank_hybrid: Function<RerankHybridCallbackArgs, Promise<Buffer>>,
) -> napi::Result<()> {
let reranker = Reranker::new(rerank_hybrid)?;
self.inner = self.inner.clone().rerank(Arc::new(reranker));
Ok(())
}
#[napi]
pub fn order_by(&mut self, ordering: Option<Vec<ColumnOrdering>>) -> napi::Result<()> {
let ordering = ordering.map(|ordering| {
ordering
.into_iter()
.map(LanceDbColumnOrdering::from)
.collect()
});
self.inner = self.inner.clone().order_by(ordering);
Ok(())
}
#[napi(catch_unwind)]
pub async fn output_schema(&self) -> napi::Result<Buffer> {
let schema = self.inner.output_schema().await.default_error()?;
schema_to_buffer(&schema)
}
#[napi(catch_unwind)]
@@ -288,6 +447,7 @@ impl VectorQuery {
&self,
max_batch_length: Option<u32>,
timeout_ms: Option<u32>,
with_row_id: Option<bool>,
) -> napi::Result<RecordBatchIterator> {
let mut execution_opts = QueryExecutionOptions::default();
if let Some(max_batch_length) = max_batch_length {
@@ -296,8 +456,12 @@ impl VectorQuery {
if let Some(timeout_ms) = timeout_ms {
execution_opts.timeout = Some(std::time::Duration::from_millis(timeout_ms as u64))
}
let inner_stream = self
.inner
let query = if with_row_id == Some(true) {
self.inner.clone().with_row_id()
} else {
self.inner.clone()
};
let inner_stream = query
.execute_with_options(execution_opts)
.await
.map_err(|e| {
@@ -320,13 +484,17 @@ impl VectorQuery {
}
#[napi(catch_unwind)]
pub async fn analyze_plan(&self) -> napi::Result<String> {
self.inner.analyze_plan().await.map_err(|e| {
napi::Error::from_reason(format!(
"Failed to execute analyze plan: {}",
convert_error(&e)
))
})
pub async fn analyze_plan(&self, distributed_metrics: Option<String>) -> napi::Result<String> {
let options = analyze_plan_options(distributed_metrics)?;
self.inner
.analyze_plan_with_options(options)
.await
.map_err(|e| {
napi::Error::from_reason(format!(
"Failed to execute analyze plan: {}",
convert_error(&e)
))
})
}
}
@@ -356,11 +524,23 @@ impl TakeQuery {
self.inner = self.inner.clone().with_row_id();
}
#[napi(js_name = "useRTE")]
pub fn use_rte(&mut self, enable: bool) {
self.inner = self.inner.clone().use_rte(enable);
}
#[napi(catch_unwind)]
pub async fn output_schema(&self) -> napi::Result<Buffer> {
let schema = self.inner.output_schema().await.default_error()?;
schema_to_buffer(&schema)
}
#[napi(catch_unwind)]
pub async fn execute(
&self,
max_batch_length: Option<u32>,
timeout_ms: Option<u32>,
with_row_id: Option<bool>,
) -> napi::Result<RecordBatchIterator> {
let mut execution_opts = QueryExecutionOptions::default();
if let Some(max_batch_length) = max_batch_length {
@@ -369,8 +549,12 @@ impl TakeQuery {
if let Some(timeout_ms) = timeout_ms {
execution_opts.timeout = Some(std::time::Duration::from_millis(timeout_ms as u64))
}
let inner_stream = self
.inner
let query = if with_row_id == Some(true) {
self.inner.clone().with_row_id()
} else {
self.inner.clone()
};
let inner_stream = query
.execute_with_options(execution_opts)
.await
.map_err(|e| {
@@ -393,13 +577,17 @@ impl TakeQuery {
}
#[napi(catch_unwind)]
pub async fn analyze_plan(&self) -> napi::Result<String> {
self.inner.analyze_plan().await.map_err(|e| {
napi::Error::from_reason(format!(
"Failed to execute analyze plan: {}",
convert_error(&e)
))
})
pub async fn analyze_plan(&self, distributed_metrics: Option<String>) -> napi::Result<String> {
let options = analyze_plan_options(distributed_metrics)?;
self.inner
.analyze_plan_with_options(options)
.await
.map_err(|e| {
napi::Error::from_reason(format!(
"Failed to execute analyze plan: {}",
convert_error(&e)
))
})
}
}
@@ -490,6 +678,7 @@ impl JsFullTextQuery {
}
#[napi(factory)]
#[allow(clippy::use_self)] // NAPI doesn't allow Self here but clippy reports it
pub fn boolean_query(queries: Vec<(String, &JsFullTextQuery)>) -> napi::Result<Self> {
let mut sub_queries = Vec::with_capacity(queries.len());
for (occur, q) in queries {
@@ -509,17 +698,22 @@ impl JsFullTextQuery {
FtsQuery::Phrase(_) => "phrase".to_string(),
FtsQuery::Boost(_) => "boost".to_string(),
FtsQuery::MultiMatch(_) => "multi_match".to_string(),
FtsQuery::CombinedFields(_) => "combined_fields".to_string(),
FtsQuery::Boolean(_) => "boolean".to_string(),
}
}
}
fn parse_fts_query(query: napi::JsObject) -> napi::Result<FullTextSearchQuery> {
if let Ok(Some(query)) = query.get::<_, &JsFullTextQuery>("query") {
fn parse_fts_query(query: Object) -> napi::Result<FullTextSearchQuery> {
// `&JsFullTextQuery` recovers a native class reference through napi's borrow-tracked
// path, which is only usable from generated `#[napi]` argument conversion. This is a
// manual lookup on a nested `Object` property instead, so use `ClassInstance`, which
// unwraps the class without requiring a borrow scope.
if let Ok(Some(query)) = query.get::<ClassInstance<JsFullTextQuery>>("query") {
Ok(FullTextSearchQuery::new_query(query.inner.clone()))
} else if let Ok(Some(query_text)) = query.get::<_, String>("query") {
} else if let Ok(Some(query_text)) = query.get::<String>("query") {
let mut query_text = query_text;
let columns = query.get::<_, Option<Vec<String>>>("columns")?.flatten();
let columns = query.get::<Option<Vec<String>>>("columns")?.flatten();
let is_phrase =
query_text.len() >= 2 && query_text.starts_with('"') && query_text.ends_with('"');
@@ -540,15 +734,12 @@ fn parse_fts_query(query: napi::JsObject) -> napi::Result<FullTextSearchQuery> {
}
};
let mut query = FullTextSearchQuery::new_query(query);
if let Some(cols) = columns {
if !cols.is_empty() {
query = query.with_columns(&cols).map_err(|e| {
napi::Error::from_reason(format!(
"Failed to set full text search columns: {}",
e
))
})?;
}
if let Some(cols) = columns
&& !cols.is_empty()
{
query = query.with_columns(&cols).map_err(|e| {
napi::Error::from_reason(format!("Failed to set full text search columns: {}", e))
})?;
}
Ok(query)
} else {
nodejs/src/remote.rs
+498
View File
@@ -3,8 +3,11 @@
use std::collections::HashMap;
use lancedb::error::Error;
use napi_derive::*;
use crate::error::NapiErrorExt;
/// Timeout configuration for remote HTTP client.
#[napi(object)]
#[derive(Debug)]
@@ -69,14 +72,43 @@ pub struct RetryConfig {
pub statuses: Option<Vec<u16>>,
}
/// TLS/mTLS configuration for the remote HTTP client.
#[napi(object)]
#[derive(Debug, Default)]
pub struct TlsConfig {
/// Path to the client certificate file (PEM format) for mTLS authentication.
pub cert_file: Option<String>,
/// Path to the client private key file (PEM format) for mTLS authentication.
pub key_file: Option<String>,
/// Path to the CA certificate file (PEM format) for server verification.
pub ssl_ca_cert: Option<String>,
/// Whether to verify the hostname in the server's certificate.
pub assert_hostname: Option<bool>,
}
#[napi(object)]
#[derive(Debug, Default)]
pub struct ClientConfig {
pub user_agent: Option<String>,
pub retry_config: Option<RetryConfig>,
pub timeout_config: Option<TimeoutConfig>,
/// Maximum number of concurrent blob HTTP requests across all tables and
/// blob handles on this connection. Defaults to 8. Can also be set with
/// `LANCE_CLIENT_BLOB_REQUEST_CONCURRENCY`. Must be greater than zero.
pub blob_request_concurrency: Option<u32>,
pub extra_headers: Option<HashMap<String, String>>,
/// The delimiter joining a namespace path and a name into one object
/// identifier. `"$"` is the only supported value, and leaving this unset is
/// how to get it; anything else is rejected when the connection is created.
pub id_delimiter: Option<String>,
pub tls_config: Option<TlsConfig>,
/// User identifier for tracking purposes.
///
/// This is sent as the `x-lancedb-user-id` header in requests to LanceDB Cloud/Enterprise.
/// It can be set directly, or via the `LANCEDB_USER_ID` environment variable.
/// Alternatively, set `LANCEDB_USER_ID_ENV_KEY` to specify another environment
/// variable that contains the user ID value.
pub user_id: Option<String>,
}
impl From<TimeoutConfig> for lancedb::remote::TimeoutConfig {
@@ -107,6 +139,291 @@ impl From<RetryConfig> for lancedb::remote::RetryConfig {
}
}
impl From<TlsConfig> for lancedb::remote::TlsConfig {
fn from(config: TlsConfig) -> Self {
Self {
cert_file: config.cert_file,
key_file: config.key_file,
ssl_ca_cert: config.ssl_ca_cert,
assert_hostname: config.assert_hostname.unwrap_or(true),
}
}
}
/// Options for the persistent OAuth token cache.
///
/// The cache is opt-in: it is only used when set as `tokenCache` on
/// `OAuthConfig`. Only refresh tokens are persisted, in a private directory
/// with owner-only permissions, so short-lived processes can reuse an
/// authenticated session instead of re-prompting on every start.
#[napi(object)]
#[derive(Clone, Debug, Default)]
pub struct TokenCacheOptions {
/// Directory that holds cached credentials. Defaults to
/// `$XDG_CACHE_HOME/lancedb/oauth`, `$HOME/.cache/lancedb/oauth` on Unix,
/// or `%LOCALAPPDATA%\lancedb\oauth` on Windows. The directory is created
/// with owner-only permissions (`0700`) when missing.
pub cache_dir: Option<String>,
/// How long to wait for the cross-process refresh lock before failing,
/// in seconds (default: 30).
pub lock_timeout_secs: Option<u32>,
}
impl From<TokenCacheOptions> for lancedb::remote::TokenCacheOptions {
fn from(options: TokenCacheOptions) -> Self {
Self {
cache_dir: options.cache_dir.map(std::path::PathBuf::from),
lock_timeout_secs: options.lock_timeout_secs.map(|secs| secs as u64),
}
}
}
/// OAuth configuration for LanceDB authentication.
///
/// This is the generated napi-rs binding shape. TypeScript users should prefer
/// the public `OAuthConfig` type exported from `@lancedb/lancedb`.
///
/// All token acquisition and refresh is handled in the Rust layer.
#[napi(object)]
#[derive(Clone)]
pub struct OAuthConfig {
/// OIDC issuer URL or OAuth authority URL.
/// For Azure: `https://login.microsoftonline.com/{tenant_id}/v2.0`
pub issuer_url: String,
/// Application / Client ID.
pub client_id: String,
/// OAuth scopes to request. For Azure managed identity, exactly one scope
/// or resource is required. For example: `["api://{app_id}/.default"]`
pub scopes: Vec<String>,
/// Optional resource indicator for authorization and token requests.
pub resource: Option<String>,
/// Optional provider-specific audience for authorization and token requests.
pub audience: Option<String>,
/// Authentication flow: "client_credentials", "authorization_code",
/// "device_code", or "azure_managed_identity"
pub flow: Option<String>,
/// Client secret (required for client_credentials).
pub client_secret: Option<String>,
/// How the client authenticates to the token endpoint: "none",
/// "client_secret_basic", or "client_secret_post". Defaults to
/// "client_secret_basic" when a client secret is set, and "none" for
/// public clients.
pub client_auth_method: Option<String>,
/// Loopback redirect URI for authorization_code.
pub redirect_uri: Option<String>,
/// Port for the authorization_code loopback callback server.
pub callback_port: Option<u16>,
/// Whether authorization_code uses S256 PKCE (default: true).
pub use_pkce: Option<bool>,
/// Client ID for user-assigned managed identity (azure_managed_identity).
pub managed_identity_client_id: Option<String>,
/// Seconds before expiry to trigger proactive refresh (default: 300).
/// Keep this well below the token TTL; if it is greater than or equal to
/// the TTL, each request refreshes the token.
pub refresh_buffer_secs: Option<u32>,
/// Opt in to the persistent token cache so short-lived processes reuse
/// one session. Only refresh tokens are persisted.
pub token_cache: Option<TokenCacheOptions>,
}
impl std::fmt::Debug for OAuthConfig {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("OAuthConfig")
.field("issuer_url", &self.issuer_url)
.field("client_id", &self.client_id)
.field("scopes", &self.scopes)
.field("resource", &self.resource)
.field("audience", &self.audience)
.field("flow", &self.flow)
.field(
"client_secret",
&self.client_secret.as_deref().map(|_| "<redacted>"),
)
.field("client_auth_method", &self.client_auth_method)
.field("redirect_uri", &self.redirect_uri)
.field("callback_port", &self.callback_port)
.field("use_pkce", &self.use_pkce)
.field(
"managed_identity_client_id",
&self.managed_identity_client_id,
)
.field("refresh_buffer_secs", &self.refresh_buffer_secs)
.field("token_cache", &self.token_cache)
.finish()
}
}
impl TryFrom<OAuthConfig> for lancedb::remote::oauth::OAuthConfig {
type Error = Error;
fn try_from(config: OAuthConfig) -> Result<Self, Self::Error> {
use lancedb::remote::oauth::{AuthorizationCodeOptions, OAuthFlow};
let flow = match config.flow.as_deref().unwrap_or("client_credentials") {
"client_credentials" => OAuthFlow::ClientCredentials,
"authorization_code" => {
let mut options =
AuthorizationCodeOptions::new().use_pkce(config.use_pkce.unwrap_or(true));
if let Some(redirect_uri) = config.redirect_uri {
options = options.redirect_uri(redirect_uri);
}
if let Some(callback_port) = config.callback_port {
options = options.callback_port(callback_port);
}
OAuthFlow::AuthorizationCode(options)
}
"device_code" => OAuthFlow::DeviceCode,
"azure_managed_identity" => OAuthFlow::AzureManagedIdentity {
client_id: config.managed_identity_client_id,
},
other => {
return Err(Error::InvalidInput {
message: format!("Unknown OAuth flow type: {other}"),
});
}
};
let client_auth_method = match config.client_auth_method.as_deref() {
Some("none") => Some(lancedb::remote::oauth::ClientAuthMethod::None),
Some("client_secret_basic") => {
Some(lancedb::remote::oauth::ClientAuthMethod::ClientSecretBasic)
}
Some("client_secret_post") => {
Some(lancedb::remote::oauth::ClientAuthMethod::ClientSecretPost)
}
None => None,
Some(other) => {
return Err(Error::InvalidInput {
message: format!("Unknown OAuth client auth method: {other}"),
});
}
};
Ok(Self {
issuer_url: config.issuer_url,
client_id: config.client_id,
client_secret: config.client_secret,
client_auth_method,
scopes: config.scopes,
resource: config.resource,
audience: config.audience,
flow,
refresh_buffer_secs: config.refresh_buffer_secs.map(|v| v as u64),
token_cache: config.token_cache.map(Into::into),
})
}
}
/// Safe, non-secret view of a cached OAuth session, returned by
/// `OAuthSession.status()` and `OAuthSession.login()`.
#[napi(object)]
#[derive(Clone, Debug)]
pub struct SessionStatus {
/// Whether a cached session exists that can obtain tokens without
/// interactive authentication.
pub refreshable: bool,
/// Canonical issuer URL of the cached session.
pub issuer_url: String,
/// Client ID of the cached session.
pub client_id: String,
/// Canonical (sorted, de-duplicated) scopes of the cached session.
pub scopes: Vec<String>,
/// Optional resource indicator for authorization and token requests.
pub resource: Option<String>,
/// Optional provider-specific audience for authorization and token requests.
pub audience: Option<String>,
/// Flow that produced the cached session.
pub flow: String,
/// When the cached session was obtained, as Unix seconds.
pub obtained_at: Option<f64>,
}
/// Result of `OAuthSession.logout()`.
#[napi(object)]
#[derive(Clone, Debug)]
pub struct SessionLogout {
/// Whether a cached credential was removed. `false` means no matching
/// session was cached; logout is idempotent.
pub removed: bool,
}
/// Explicit OAuth session lifecycle for the persistent token cache: eager
/// `login`, non-secret `status`, and local `logout`.
///
/// A session is built from the same `OAuthConfig` used to connect (including
/// its `tokenCache` options). A connection created with the same
/// configuration shares the cache, so logging in here prepares tokens for
/// later processes without any database request.
#[napi]
pub struct OAuthSession {
inner: lancedb::remote::OAuthSession,
}
#[napi]
impl OAuthSession {
/// Create a session manager for the given OAuth configuration.
///
/// The configuration must enable `tokenCache` options and use a flow that
/// supports persistent sessions (authorization code or device code).
#[napi(constructor)]
pub fn new(config: OAuthConfig) -> napi::Result<Self> {
let config: lancedb::remote::oauth::OAuthConfig = config.try_into().default_error()?;
let inner = lancedb::remote::OAuthSession::new(config).default_error()?;
Ok(Self { inner })
}
/// Eagerly run the configured authentication flow and store the session.
///
/// A successful login always replaces any prior cached session for this
/// identity; if the provider does not issue a refresh token (for example
/// without `offline_access`), the previous record is removed and the
/// status reports `refreshable == false`.
#[napi(catch_unwind)]
pub async fn login(&self) -> napi::Result<SessionStatus> {
let status = self.inner.login().await.default_error()?;
Ok(SessionStatus::from(status))
}
/// Report whether a matching cached session exists, with safe metadata.
///
/// This never contacts the identity provider and never exposes token
/// values.
#[napi(catch_unwind)]
pub async fn status(&self) -> napi::Result<SessionStatus> {
let status = self.inner.status().await.default_error()?;
Ok(SessionStatus::from(status))
}
/// Remove the matching local cached credential.
///
/// This only deletes the local cache entry. It does not revoke the
/// refresh token with the provider and does not sign out of a browser
/// SSO session. Repeated calls succeed; `removed` reports whether a
/// credential existed.
#[napi(catch_unwind)]
pub async fn logout(&self) -> napi::Result<SessionLogout> {
let logout = self.inner.logout().await.default_error()?;
Ok(SessionLogout {
removed: logout.removed,
})
}
}
impl From<lancedb::remote::SessionStatus> for SessionStatus {
fn from(status: lancedb::remote::SessionStatus) -> Self {
Self {
refreshable: status.refreshable,
issuer_url: status.issuer_url,
client_id: status.client_id,
scopes: status.scopes,
resource: status.resource,
audience: status.audience,
flow: status.flow,
obtained_at: status.obtained_at.map(|secs| secs as f64),
}
}
}
impl From<ClientConfig> for lancedb::remote::ClientConfig {
fn from(config: ClientConfig) -> Self {
Self {
@@ -115,8 +432,189 @@ impl From<ClientConfig> for lancedb::remote::ClientConfig {
.unwrap_or(concat!("LanceDB-Node-Client/", env!("CARGO_PKG_VERSION")).to_string()),
retry_config: config.retry_config.map(Into::into).unwrap_or_default(),
timeout_config: config.timeout_config.map(Into::into).unwrap_or_default(),
blob_request_concurrency: config.blob_request_concurrency.map(|value| value as usize),
extra_headers: config.extra_headers.unwrap_or_default(),
id_delimiter: config.id_delimiter,
tls_config: config.tls_config.map(Into::into),
header_provider: None, // the header provider is set separately later
user_id: config.user_id,
// Resolved from LANCE_CLIENT_MAX_BYTES_PER_REQUEST or the default.
max_bytes_per_request: None,
// Resolved from LANCE_CLIENT_MAX_REQUEST_DURATION or the read timeout.
max_request_duration: None,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_unknown_oauth_flow_returns_invalid_input() {
let config = OAuthConfig {
issuer_url: "https://issuer.example.com".to_string(),
client_id: "client-id".to_string(),
scopes: vec!["scope".to_string()],
flow: Some("typo".to_string()),
client_secret: None,
client_auth_method: None,
redirect_uri: None,
callback_port: None,
use_pkce: None,
managed_identity_client_id: None,
refresh_buffer_secs: None,
resource: None,
audience: None,
token_cache: None,
};
let err = lancedb::remote::oauth::OAuthConfig::try_from(config).unwrap_err();
assert!(matches!(
err,
Error::InvalidInput { message }
if message == "Unknown OAuth flow type: typo"
));
}
#[test]
fn test_oauth_config_debug_redacts_client_secret() {
let config = OAuthConfig {
issuer_url: "https://issuer.example.com".to_string(),
client_id: "client-id".to_string(),
scopes: vec!["scope".to_string()],
flow: Some("client_credentials".to_string()),
client_secret: Some("super-secret".to_string()),
client_auth_method: None,
redirect_uri: None,
callback_port: None,
use_pkce: None,
managed_identity_client_id: None,
refresh_buffer_secs: None,
resource: None,
audience: None,
token_cache: None,
};
let debug = format!("{config:?}");
assert!(!debug.contains("super-secret"));
assert!(debug.contains("client_secret: Some(\"<redacted>\")"));
}
#[test]
fn test_authorization_code_conversion_preserves_options() {
let config = OAuthConfig {
issuer_url: "https://issuer.example.com".to_string(),
client_id: "client-id".to_string(),
scopes: vec!["openid".to_string()],
flow: Some("authorization_code".to_string()),
client_secret: Some("secret".to_string()),
client_auth_method: None,
redirect_uri: Some("http://127.0.0.1:9000/callback".to_string()),
callback_port: Some(9000),
use_pkce: Some(false),
managed_identity_client_id: None,
refresh_buffer_secs: None,
resource: Some("urn:resource".into()),
audience: Some("audience".into()),
token_cache: None,
};
let converted = lancedb::remote::oauth::OAuthConfig::try_from(config).unwrap();
let lancedb::remote::oauth::OAuthFlow::AuthorizationCode(options) = converted.flow else {
panic!("expected authorization code flow");
};
assert_eq!(
options.redirect_uri.as_deref(),
Some("http://127.0.0.1:9000/callback")
);
assert_eq!(options.callback_port, Some(9000));
assert!(!options.use_pkce);
assert_eq!(converted.resource.as_deref(), Some("urn:resource"));
assert_eq!(converted.audience.as_deref(), Some("audience"));
}
#[test]
fn test_device_code_conversion() {
let config = OAuthConfig {
issuer_url: "https://issuer.example.com".to_string(),
client_id: "client-id".to_string(),
scopes: vec!["openid".to_string()],
flow: Some("device_code".to_string()),
client_secret: None,
client_auth_method: None,
redirect_uri: None,
callback_port: None,
use_pkce: None,
managed_identity_client_id: None,
refresh_buffer_secs: None,
resource: None,
audience: None,
token_cache: None,
};
let converted = lancedb::remote::oauth::OAuthConfig::try_from(config).unwrap();
assert!(matches!(
converted.flow,
lancedb::remote::oauth::OAuthFlow::DeviceCode
));
}
#[test]
fn test_client_auth_method_conversion() {
use lancedb::remote::oauth::ClientAuthMethod;
for (value, expected) in [
("none", ClientAuthMethod::None),
("client_secret_basic", ClientAuthMethod::ClientSecretBasic),
("client_secret_post", ClientAuthMethod::ClientSecretPost),
] {
let config = OAuthConfig {
issuer_url: "https://issuer.example.com".to_string(),
client_id: "client-id".to_string(),
scopes: vec!["openid".to_string()],
flow: Some("device_code".to_string()),
client_secret: None,
client_auth_method: Some(value.to_string()),
resource: None,
audience: None,
redirect_uri: None,
callback_port: None,
use_pkce: None,
managed_identity_client_id: None,
refresh_buffer_secs: None,
token_cache: None,
};
let converted = lancedb::remote::oauth::OAuthConfig::try_from(config).unwrap();
assert_eq!(converted.client_auth_method, Some(expected));
}
}
#[test]
fn test_unknown_client_auth_method_returns_invalid_input() {
let config = OAuthConfig {
issuer_url: "https://issuer.example.com".to_string(),
client_id: "client-id".to_string(),
scopes: vec!["openid".to_string()],
flow: Some("device_code".to_string()),
client_secret: None,
client_auth_method: Some("typo".to_string()),
resource: None,
audience: None,
redirect_uri: None,
callback_port: None,
use_pkce: None,
managed_identity_client_id: None,
refresh_buffer_secs: None,
token_cache: None,
};
let err = lancedb::remote::oauth::OAuthConfig::try_from(config).unwrap_err();
assert!(matches!(
err,
Error::InvalidInput { message }
if message == "Unknown OAuth client auth method: typo"
));
}
}
nodejs/src/rerankers.rs
+26 -29
View File
@@ -3,10 +3,7 @@
use arrow_array::RecordBatch;
use async_trait::async_trait;
use napi::{
bindgen_prelude::*,
threadsafe_function::{ErrorStrategy, ThreadsafeFunction},
};
use napi::{bindgen_prelude::*, threadsafe_function::ThreadsafeFunction};
use napi_derive::napi;
use lancedb::ipc::batches_to_ipc_file;
@@ -15,27 +12,32 @@ use lancedb::{error::Error, ipc::ipc_file_to_batches};
use crate::error::NapiErrorExt;
type RerankHybridFn = ThreadsafeFunction<
RerankHybridCallbackArgs,
Promise<Buffer>,
RerankHybridCallbackArgs,
Status,
false,
true,
>;
/// Reranker implementation that "wraps" a NodeJS Reranker implementation.
/// This contains references to the callbacks that can be used to invoke the
/// reranking methods on the NodeJS implementation and handles serializing the
/// record batches to Arrow IPC buffers.
#[napi]
pub struct Reranker {
/// callback to the Javascript which will call the rerankHybrid method of
/// some Reranker implementation
rerank_hybrid: ThreadsafeFunction<RerankHybridCallbackArgs, ErrorStrategy::CalleeHandled>,
rerank_hybrid: RerankHybridFn,
}
#[napi]
impl Reranker {
#[napi]
pub fn new(callbacks: RerankerCallbacks) -> Self {
let rerank_hybrid = callbacks
.rerank_hybrid
.create_threadsafe_function(0, move |ctx| Ok(vec![ctx.value]))
.unwrap();
Self { rerank_hybrid }
pub fn new(
rerank_hybrid: Function<RerankHybridCallbackArgs, Promise<Buffer>>,
) -> napi::Result<Self> {
let rerank_hybrid = rerank_hybrid
.build_threadsafe_function()
.weak::<true>()
.build()?;
Ok(Self { rerank_hybrid })
}
}
@@ -49,16 +51,16 @@ impl lancedb::rerankers::Reranker for Reranker {
) -> lancedb::error::Result<RecordBatch> {
let callback_args = RerankHybridCallbackArgs {
query: query.to_string(),
vec_results: batches_to_ipc_file(&[vector_results])?,
fts_results: batches_to_ipc_file(&[fts_results])?,
vec_results: Buffer::from(batches_to_ipc_file(&[vector_results])?.as_ref()),
fts_results: Buffer::from(batches_to_ipc_file(&[fts_results])?.as_ref()),
};
let promised_buffer: Promise<Buffer> = self
.rerank_hybrid
.call_async(Ok(callback_args))
.call_async(callback_args)
.await
.map_err(|e| Error::Runtime {
message: format!("napi error status={}, reason={}", e.status, e.reason),
})?;
message: format!("napi error status={}, reason={}", e.status, e.reason),
})?;
let buffer = promised_buffer.await.map_err(|e| Error::Runtime {
message: format!("napi error status={}, reason={}", e.status, e.reason),
})?;
@@ -77,16 +79,11 @@ impl std::fmt::Debug for Reranker {
}
}
#[napi(object)]
pub struct RerankerCallbacks {
pub rerank_hybrid: JsFunction,
}
#[napi(object)]
pub struct RerankHybridCallbackArgs {
pub query: String,
pub vec_results: Vec<u8>,
pub fts_results: Vec<u8>,
pub vec_results: Buffer,
pub fts_results: Buffer,
}
fn buffer_to_record_batch(buffer: Buffer) -> Result<RecordBatch> {
nodejs/src/scannable.rs
+253
View File
@@ -0,0 +1,253 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
//! NodeJS binding for the [`lancedb::data::scannable::Scannable`] trait.
//!
//! The JS side supplies a `getNextBatch(isStart)` callback that returns the
//! next Arrow `RecordBatch` encoded as a self-contained Arrow IPC Stream
//! message (schema message + record batch message + EOS marker) wrapped in a
//! `Buffer`, or `null` when the stream is exhausted. The Rust side parses
//! each buffer with `arrow_ipc::reader::StreamReader`, validates every
//! standalone batch stream against the declared schema, and yields decoded
//! `RecordBatch`es as a [`SendableRecordBatchStream`].
//!
//! `isStart` is `true` on the first `getNextBatch` call of each new
//! `scan_as_stream` and `false` thereafter. JS uses it to drop any cached
//! iterator and re-invoke its factory at scan boundaries, so retries
//! triggered by mid-stream failures restart at batch 0.
use std::io::Cursor;
use std::sync::Arc;
use arrow_array::RecordBatch;
use arrow_ipc::reader::StreamReader;
use arrow_schema::SchemaRef;
use futures::stream::once;
use lancedb::arrow::{SendableRecordBatchStream, SimpleRecordBatchStream};
use lancedb::data::scannable::Scannable as LanceScannable;
use lancedb::ipc::ipc_file_to_schema;
use lancedb::{Error, Result as LanceResult};
use napi::bindgen_prelude::*;
use napi::threadsafe_function::ThreadsafeFunction;
use napi_derive::napi;
/// Threadsafe handle to the JS `getNextBatch` callback. The callback takes a
/// single boolean `isStart` (`true` on the first call of each new scan) and
/// returns a Promise that resolves to a `Buffer` containing one IPC Stream
/// message, or `null` at end-of-stream.
type GetNextBatchFn = ThreadsafeFunction<bool, Promise<Option<Buffer>>, bool, Status, false>;
/// A Rust-side view of a JS-constructed `Scannable`.
///
/// Held in JS as the return value of the `Scannable` class constructor. When
/// passed to a consumer that accepts `impl lancedb::data::scannable::Scannable`,
/// the consumer invokes `scan_as_stream()` to pull batches through the JS
/// callback.
#[napi]
pub struct NapiScannable {
schema: SchemaRef,
num_rows: Option<usize>,
rescannable: bool,
// `ThreadsafeFunction` is not `Clone`; wrap in `Arc` so the stream
// returned by `scan_as_stream` can own a handle independent of `self`.
get_next_batch: Arc<GetNextBatchFn>,
// Tracks whether a scan has already started; used to enforce one-shot
// semantics on non-rescannable sources.
scanned: bool,
}
#[napi]
impl NapiScannable {
/// Construct a new `NapiScannable`.
///
/// - `schema_buf` — Arrow IPC File buffer carrying only the schema (no batches).
/// - `num_rows` — optional row count hint; not validated against the stream.
/// - `rescannable` — whether `get_next_batch` may be re-driven after the
/// scan completes.
/// - `get_next_batch` -- JS callback that yields the next batch as an Arrow
/// IPC Stream message wrapped in a `Buffer`, or `null` at EOF. The
/// `isStart` argument is `true` on the first call of each new scan;
/// JS uses it to discard any cached iterator before pulling.
#[napi(constructor)]
pub fn new(
schema_buf: Buffer,
num_rows: Option<i64>,
rescannable: bool,
get_next_batch: Function<bool, Promise<Option<Buffer>>>,
) -> napi::Result<Self> {
let schema = ipc_file_to_schema(schema_buf.to_vec())
.map_err(|e| napi::Error::from_reason(format!("Invalid schema buffer: {}", e)))?;
let num_rows = num_rows
.map(|n| {
usize::try_from(n)
.map_err(|_| napi::Error::from_reason("num_rows must be non-negative"))
})
.transpose()?;
let get_next_batch = Arc::new(get_next_batch.build_threadsafe_function().build()?);
Ok(Self {
schema,
num_rows,
rescannable,
get_next_batch,
scanned: false,
})
}
}
impl std::fmt::Debug for NapiScannable {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("NapiScannable")
.field("schema", &self.schema)
.field("num_rows", &self.num_rows)
.field("rescannable", &self.rescannable)
.finish()
}
}
impl LanceScannable for NapiScannable {
fn schema(&self) -> SchemaRef {
self.schema.clone()
}
fn scan_as_stream(&mut self) -> SendableRecordBatchStream {
let schema = self.schema.clone();
// One-shot enforcement for non-rescannable sources: return a stream
// whose first item is an error.
if self.scanned && !self.rescannable {
let err_stream = once(async {
Err(Error::InvalidInput {
message: "Scannable has already been consumed (non-rescannable source)"
.to_string(),
})
});
return Box::pin(SimpleRecordBatchStream::new(err_stream, schema));
}
self.scanned = true;
let tsfn = Arc::clone(&self.get_next_batch);
let declared_schema = schema.clone();
// State threaded through the unfold. `is_first_pull` starts true so
// the first call into JS signals a new-scan boundary; JS uses it to
// reset any cached iterator before factory()-ing a fresh one.
let initial = State {
tsfn,
batch_index: 0,
declared_schema,
errored: false,
is_first_pull: true,
};
let stream = futures::stream::unfold(initial, |mut state| async move {
if state.errored {
return None;
}
// Pull the next IPC Stream buffer from JS. `is_first_pull` is
// consumed here and cleared so subsequent pulls continue the
// same scan rather than restarting it.
let is_start = state.is_first_pull;
state.is_first_pull = false;
let buf = match pull_next(&state.tsfn, is_start).await {
Ok(Some(buf)) => buf,
Ok(None) => return None,
Err(e) => {
state.errored = true;
return Some((Err(e), state));
}
};
match decode_one_batch(buf.as_ref(), &state.declared_schema) {
Ok(batch) => {
state.batch_index += 1;
Some((Ok(batch), state))
}
Err(e) => {
let tagged = Error::Runtime {
message: format!(
"[scannable/rust-bridge] failure at batch index {}: {}",
state.batch_index, e
),
};
state.errored = true;
Some((Err(tagged), state))
}
}
});
Box::pin(SimpleRecordBatchStream::new(stream, schema))
}
fn num_rows(&self) -> Option<usize> {
self.num_rows
}
fn rescannable(&self) -> bool {
self.rescannable
}
}
struct State {
tsfn: Arc<GetNextBatchFn>,
batch_index: usize,
declared_schema: SchemaRef,
errored: bool,
/// True for the very first pull of a new scan. Forwarded to JS so the
/// callback can drop any cached iterator and call its factory fresh,
/// which makes rescannable sources restart at batch 0 even when the
/// previous scan ended mid-stream.
is_first_pull: bool,
}
/// Invoke the JS callback and await its Promise. `is_start` is forwarded to
/// the JS side as the `isStart` argument so it can reset its iterator at the
/// scan boundary. Errors on the JS side surface here as rejected promises
/// and are tunneled back as `lancedb::Error::Runtime`.
async fn pull_next(tsfn: &GetNextBatchFn, is_start: bool) -> LanceResult<Option<Buffer>> {
let promise = tsfn
.call_async(is_start)
.await
.map_err(|e| Error::Runtime {
message: format!(
"[scannable/js-factory] napi error status={}, reason={}",
e.status, e.reason
),
})?;
promise.await.map_err(|e| Error::Runtime {
message: format!(
"[scannable/js-iterator] napi error status={}, reason={}",
e.status, e.reason
),
})
}
/// Decode one IPC Stream buffer (schema + batch + EOS) into a `RecordBatch`.
/// Each buffer is a standalone IPC stream, so every decoded stream schema must
/// match the one declared at construction.
fn decode_one_batch(buf: &[u8], declared: &SchemaRef) -> LanceResult<RecordBatch> {
let reader = StreamReader::try_new(Cursor::new(buf), None).map_err(|e| Error::Runtime {
message: format!("failed to open IPC stream reader: {}", e),
})?;
let actual = reader.schema();
if actual.as_ref() != declared.as_ref() {
return Err(Error::InvalidInput {
message: format!(
"declared schema does not match stream schema: declared={:?} actual={:?}",
declared, actual
),
});
}
let mut iter = reader;
let batch = iter
.next()
.ok_or_else(|| Error::Runtime {
message: "IPC stream contained schema but no record batch".to_string(),
})?
.map_err(|e| Error::Runtime {
message: format!("failed to decode record batch: {}", e),
})?;
Ok(batch)
}
nodejs/src/session.rs
+3 -4
View File
@@ -94,9 +94,8 @@ impl napi::bindgen_prelude::FromNapiValue for Session {
env: napi::sys::napi_env,
napi_val: napi::sys::napi_value,
) -> napi::Result<Self> {
let object: napi::bindgen_prelude::ClassInstance<Session> =
napi::bindgen_prelude::ClassInstance::from_napi_value(env, napi_val)?;
let copy = object.clone();
Ok(copy)
let object: napi::bindgen_prelude::ClassInstance<Self> =
unsafe { napi::bindgen_prelude::ClassInstance::from_napi_value(env, napi_val)? };
Ok((*object).clone())
}
}
nodejs/src/table.rs
+1048 -52
View File
File diff suppressed because it is too large Load Diff
nodejs/src/util.rs
+15
View File
@@ -1,7 +1,10 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use arrow_ipc::writer::FileWriter;
use arrow_schema::Schema;
use lancedb::DistanceType;
use napi::bindgen_prelude::Buffer;
pub fn parse_distance_type(distance_type: impl AsRef<str>) -> napi::Result<DistanceType> {
match distance_type.as_ref().to_lowercase().as_str() {
@@ -15,3 +18,15 @@ pub fn parse_distance_type(distance_type: impl AsRef<str>) -> napi::Result<Dista
))),
}
}
/// Convert an Arrow Schema to an Arrow IPC file buffer
pub fn schema_to_buffer(schema: &Schema) -> napi::Result<Buffer> {
let mut writer = FileWriter::try_new(vec![], schema)
.map_err(|e| napi::Error::from_reason(format!("Failed to create IPC file: {}", e)))?;
writer
.finish()
.map_err(|e| napi::Error::from_reason(format!("Failed to finish IPC file: {}", e)))?;
Ok(Buffer::from(writer.into_inner().map_err(|e| {
napi::Error::from_reason(format!("Failed to get IPC file: {}", e))
})?))
}
nodejs/tsconfig.json
+4 -1
View File
@@ -10,7 +10,10 @@
"resolveJsonModule": true,
"emitDecoratorMetadata": true,
"experimentalDecorators": true,
"moduleResolution": "Node"
"moduleResolution": "Node",
// flatbuffers 25 (via apache-arrow 21) ships declarations that need
// TypeScript 5.7+ generic typed arrays; skip checking third-party .d.ts.
"skipLibCheck": true
},
"exclude": ["./dist/*", "./examples/*"],
"typedocOptions": {
nodejs/typedoc.json
+2 -1
View File
@@ -4,7 +4,8 @@
"lancedb/native.d.ts:VectorQuery",
"lancedb/native.d.ts:TakeQuery",
"lancedb/native.d.ts:RecordBatchIterator",
"lancedb/native.d.ts:NativeMergeInsertBuilder"
"lancedb/native.d.ts:NativeMergeInsertBuilder",
"lancedb/native.d.ts:TokenCacheOptions"
],
"useHTMLEncodedBrackets": true,
"useCodeBlocks": true,