mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-30 01:48:19 +00:00
Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 32c77879c9 | |||
| 9330a9b851 | |||
| 02de07576e | |||
| 81617fd3d9 |
@@ -45,7 +45,9 @@ jobs:
|
|||||||
- name: Set up Node.js
|
- name: Set up Node.js
|
||||||
uses: actions/setup-node@v4
|
uses: actions/setup-node@v4
|
||||||
with:
|
with:
|
||||||
node-version: 20
|
# pnpm 11 (used by the nodejs install step below) requires
|
||||||
|
# Node >= 22.13; use 24 since 22 hits EOL in October.
|
||||||
|
node-version: 24
|
||||||
|
|
||||||
- name: Install Codex CLI
|
- name: Install Codex CLI
|
||||||
run: npm install -g @openai/codex
|
run: npm install -g @openai/codex
|
||||||
@@ -79,10 +81,14 @@ jobs:
|
|||||||
java-version: '11'
|
java-version: '11'
|
||||||
cache: maven
|
cache: maven
|
||||||
|
|
||||||
|
- name: Setup pnpm
|
||||||
|
uses: pnpm/action-setup@v4
|
||||||
|
with:
|
||||||
|
version: 11.1.1
|
||||||
- name: Install Node.js dependencies for TypeScript bindings
|
- name: Install Node.js dependencies for TypeScript bindings
|
||||||
run: |
|
run: |
|
||||||
cd nodejs
|
cd nodejs
|
||||||
npm ci
|
pnpm install --frozen-lockfile
|
||||||
|
|
||||||
- name: Configure git user
|
- name: Configure git user
|
||||||
run: |
|
run: |
|
||||||
@@ -137,7 +143,7 @@ jobs:
|
|||||||
- For Rust test failures: Run the specific test with "cargo test -p <crate> <test_name>"
|
- For Rust test failures: Run the specific test with "cargo test -p <crate> <test_name>"
|
||||||
- For Python test failures: Build with "cd python && maturin develop" then run "pytest <specific_test_file>::<test_name>"
|
- For Python test failures: Build with "cd python && maturin develop" then run "pytest <specific_test_file>::<test_name>"
|
||||||
- For Java test failures: Run "cd java && mvn test -Dtest=<TestClass>#<testMethod>"
|
- For Java test failures: Run "cd java && mvn test -Dtest=<TestClass>#<testMethod>"
|
||||||
- For TypeScript test failures: Run "cd nodejs && npm run build && npm test -- --testNamePattern='<test_name>'"
|
- For TypeScript test failures: Run "cd nodejs && pnpm build && pnpm test -- --testNamePattern='<test_name>'"
|
||||||
- Do NOT run the full test suite - only run the tests that were failing
|
- Do NOT run the full test suite - only run the tests that were failing
|
||||||
|
|
||||||
7. If the additional guidelines are provided, follow them as well.
|
7. If the additional guidelines are provided, follow them as well.
|
||||||
|
|||||||
@@ -42,11 +42,17 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
lfs: true
|
lfs: true
|
||||||
|
- uses: pnpm/action-setup@v4
|
||||||
|
with:
|
||||||
|
version: 11.1.1
|
||||||
- uses: actions/setup-node@v4
|
- uses: actions/setup-node@v4
|
||||||
with:
|
with:
|
||||||
node-version: 20
|
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
|
||||||
cache: 'npm'
|
# in October. The library itself still supports Node >= 18
|
||||||
cache-dependency-path: nodejs/package-lock.json
|
# (see test matrix below).
|
||||||
|
node-version: 24
|
||||||
|
cache: 'pnpm'
|
||||||
|
cache-dependency-path: nodejs/pnpm-lock.yaml
|
||||||
- uses: actions-rust-lang/setup-rust-toolchain@v1
|
- uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||||
with:
|
with:
|
||||||
components: rustfmt, clippy
|
components: rustfmt, clippy
|
||||||
@@ -61,11 +67,13 @@ jobs:
|
|||||||
run: cargo clippy --profile ci --all --all-features -- -D warnings
|
run: cargo clippy --profile ci --all --all-features -- -D warnings
|
||||||
- name: Lint Typescript
|
- name: Lint Typescript
|
||||||
run: |
|
run: |
|
||||||
npm ci
|
pnpm install --frozen-lockfile
|
||||||
npm run lint-ci
|
pnpm lint-ci
|
||||||
- name: Lint examples
|
- name: Lint examples
|
||||||
working-directory: nodejs/examples
|
working-directory: nodejs/examples
|
||||||
run: npm ci && npm run lint-ci
|
# The `@lancedb/lancedb` dep points at file:../dist; pnpm errors if
|
||||||
|
# that dir is missing, so create an empty one for lint-only runs.
|
||||||
|
run: mkdir -p ../dist && pnpm install --frozen-lockfile && pnpm lint-ci
|
||||||
linux:
|
linux:
|
||||||
name: Linux (NodeJS ${{ matrix.node-version }})
|
name: Linux (NodeJS ${{ matrix.node-version }})
|
||||||
timeout-minutes: 30
|
timeout-minutes: 30
|
||||||
@@ -82,14 +90,18 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
lfs: true
|
lfs: true
|
||||||
- uses: actions/setup-node@v4
|
- uses: pnpm/action-setup@v4
|
||||||
name: Setup Node.js 20 for build
|
|
||||||
with:
|
with:
|
||||||
# @napi-rs/cli v3 requires Node >= 20.12 (via @inquirer/prompts@8).
|
version: 11.1.1
|
||||||
# Build always on Node 20; tests run on the matrix version below.
|
- uses: actions/setup-node@v4
|
||||||
node-version: 20
|
name: Setup Node.js 24 for build
|
||||||
cache: 'npm'
|
with:
|
||||||
cache-dependency-path: nodejs/package-lock.json
|
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
|
||||||
|
# in October. Build/install runs on Node 24; tests run on the
|
||||||
|
# matrix version below using direct jest invocation.
|
||||||
|
node-version: 24
|
||||||
|
cache: 'pnpm'
|
||||||
|
cache-dependency-path: nodejs/pnpm-lock.yaml
|
||||||
- uses: Swatinem/rust-cache@v2
|
- uses: Swatinem/rust-cache@v2
|
||||||
- name: Install dependencies
|
- name: Install dependencies
|
||||||
run: |
|
run: |
|
||||||
@@ -97,45 +109,52 @@ jobs:
|
|||||||
sudo apt install -y protobuf-compiler libssl-dev
|
sudo apt install -y protobuf-compiler libssl-dev
|
||||||
- name: Build
|
- name: Build
|
||||||
run: |
|
run: |
|
||||||
npm ci --include=optional
|
pnpm install --frozen-lockfile
|
||||||
npm run build:debug -- --profile ci
|
# No `--` separator: pnpm forwards it literally, which would
|
||||||
|
# make napi-rs treat `--profile ci` as a cargo passthrough arg.
|
||||||
|
pnpm build:debug --profile ci
|
||||||
|
pnpm tsc
|
||||||
|
- name: Setup examples
|
||||||
|
working-directory: nodejs/examples
|
||||||
|
run: pnpm install --frozen-lockfile
|
||||||
|
- name: Check docs
|
||||||
|
run: |
|
||||||
|
# We run this as part of the job because the binary needs to be built
|
||||||
|
# first to export the types of the native code.
|
||||||
|
set -e
|
||||||
|
# `pnpm docs` would invoke pnpm's built-in `docs` command, not
|
||||||
|
# the script — use `pnpm run docs`.
|
||||||
|
pnpm run docs
|
||||||
|
if ! git diff --exit-code -- ../ ':(exclude)Cargo.lock'; then
|
||||||
|
echo "Docs need to be updated"
|
||||||
|
echo "Run 'pnpm run docs', fix any warnings, and commit the changes."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
- uses: actions/setup-node@v4
|
- uses: actions/setup-node@v4
|
||||||
name: Setup Node.js ${{ matrix.node-version }} for test
|
name: Setup Node.js ${{ matrix.node-version }} for test
|
||||||
with:
|
with:
|
||||||
node-version: ${{ matrix.node-version }}
|
node-version: ${{ matrix.node-version }}
|
||||||
- name: Compile TypeScript
|
|
||||||
run: npm run tsc
|
|
||||||
- name: Setup localstack
|
- name: Setup localstack
|
||||||
working-directory: .
|
working-directory: .
|
||||||
run: docker compose up --detach --wait
|
run: docker compose up --detach --wait
|
||||||
- name: Test
|
- name: Test
|
||||||
env:
|
env:
|
||||||
S3_TEST: "1"
|
S3_TEST: "1"
|
||||||
run: npm run test
|
# Newer @smithy/core uses dynamic ESM imports.
|
||||||
- name: Setup examples
|
NODE_OPTIONS: "--experimental-vm-modules"
|
||||||
working-directory: nodejs/examples
|
# Invoke jest directly because pnpm 11 itself requires Node 22+
|
||||||
run: npm ci
|
# while the matrix tests on older Node versions.
|
||||||
|
run: npx jest --verbose
|
||||||
- name: Test examples
|
- name: Test examples
|
||||||
working-directory: ./
|
working-directory: ./
|
||||||
env:
|
env:
|
||||||
OPENAI_API_KEY: test
|
OPENAI_API_KEY: test
|
||||||
OPENAI_BASE_URL: http://0.0.0.0:8000
|
OPENAI_BASE_URL: http://0.0.0.0:8000
|
||||||
|
NODE_OPTIONS: "--experimental-vm-modules"
|
||||||
run: |
|
run: |
|
||||||
python ci/mock_openai.py &
|
python ci/mock_openai.py &
|
||||||
cd nodejs/examples
|
cd nodejs/examples
|
||||||
npm test
|
npx jest --testEnvironment jest-environment-node-single-context --verbose
|
||||||
- name: Check docs
|
|
||||||
run: |
|
|
||||||
# We run this as part of the job because the binary needs to be built
|
|
||||||
# first to export the types of the native code.
|
|
||||||
set -e
|
|
||||||
npm ci
|
|
||||||
npm run docs
|
|
||||||
if ! git diff --exit-code -- ../ ':(exclude)Cargo.lock'; then
|
|
||||||
echo "Docs need to be updated"
|
|
||||||
echo "Run 'npm run docs', fix any warnings, and commit the changes."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
macos:
|
macos:
|
||||||
timeout-minutes: 30
|
timeout-minutes: 30
|
||||||
runs-on: "macos-14"
|
runs-on: "macos-14"
|
||||||
@@ -148,20 +167,28 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
lfs: true
|
lfs: true
|
||||||
|
- uses: pnpm/action-setup@v4
|
||||||
|
with:
|
||||||
|
version: 11.1.1
|
||||||
- uses: actions/setup-node@v4
|
- uses: actions/setup-node@v4
|
||||||
with:
|
with:
|
||||||
node-version: 20
|
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
|
||||||
cache: 'npm'
|
# in October.
|
||||||
cache-dependency-path: nodejs/package-lock.json
|
node-version: 24
|
||||||
|
cache: 'pnpm'
|
||||||
|
cache-dependency-path: nodejs/pnpm-lock.yaml
|
||||||
|
- uses: dtolnay/rust-toolchain@stable
|
||||||
- uses: Swatinem/rust-cache@v2
|
- uses: Swatinem/rust-cache@v2
|
||||||
- name: Install dependencies
|
- name: Install dependencies
|
||||||
run: |
|
run: |
|
||||||
brew install protobuf
|
brew install protobuf
|
||||||
- name: Build
|
- name: Build
|
||||||
run: |
|
run: |
|
||||||
npm ci --include=optional
|
pnpm install --frozen-lockfile
|
||||||
npm run build:debug -- --profile ci
|
# No `--` separator: pnpm forwards it literally, which would
|
||||||
npm run tsc
|
# make napi-rs treat `--profile ci` as a cargo passthrough arg.
|
||||||
|
pnpm build:debug --profile ci
|
||||||
|
pnpm tsc
|
||||||
- name: Test
|
- name: Test
|
||||||
run: |
|
run: |
|
||||||
npm run test
|
pnpm test
|
||||||
|
|||||||
@@ -171,13 +171,18 @@ jobs:
|
|||||||
working-directory: nodejs
|
working-directory: nodejs
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
|
- name: Setup pnpm
|
||||||
|
uses: pnpm/action-setup@v4
|
||||||
|
with:
|
||||||
|
version: 11.1.1
|
||||||
- name: Setup node
|
- name: Setup node
|
||||||
uses: actions/setup-node@v4
|
uses: actions/setup-node@v4
|
||||||
if: ${{ !matrix.settings.docker }}
|
|
||||||
with:
|
with:
|
||||||
node-version: 20
|
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
|
||||||
cache: npm
|
# in October.
|
||||||
cache-dependency-path: nodejs/package-lock.json
|
node-version: 24
|
||||||
|
cache: pnpm
|
||||||
|
cache-dependency-path: nodejs/pnpm-lock.yaml
|
||||||
- name: Install
|
- name: Install
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@stable
|
||||||
if: ${{ !matrix.settings.docker }}
|
if: ${{ !matrix.settings.docker }}
|
||||||
@@ -195,7 +200,7 @@ jobs:
|
|||||||
target/
|
target/
|
||||||
key: nodejs-${{ matrix.settings.target }}-cargo-${{ matrix.settings.host }}
|
key: nodejs-${{ matrix.settings.target }}-cargo-${{ matrix.settings.host }}
|
||||||
- name: Install dependencies
|
- name: Install dependencies
|
||||||
run: npm ci
|
run: pnpm install --frozen-lockfile
|
||||||
- name: Install Zig
|
- name: Install Zig
|
||||||
uses: mlugg/setup-zig@v2
|
uses: mlugg/setup-zig@v2
|
||||||
if: ${{ contains(matrix.settings.target, 'musl') }}
|
if: ${{ contains(matrix.settings.target, 'musl') }}
|
||||||
@@ -248,7 +253,7 @@ jobs:
|
|||||||
# one to do the upload.
|
# one to do the upload.
|
||||||
- name: Make generic artifacts
|
- name: Make generic artifacts
|
||||||
if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }}
|
if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }}
|
||||||
run: npm run tsc
|
run: pnpm tsc
|
||||||
- name: Upload Generic Artifacts
|
- name: Upload Generic Artifacts
|
||||||
if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }}
|
if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }}
|
||||||
uses: actions/upload-artifact@v4
|
uses: actions/upload-artifact@v4
|
||||||
@@ -283,14 +288,24 @@ jobs:
|
|||||||
working-directory: nodejs
|
working-directory: nodejs
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
- name: Setup node
|
- name: Setup pnpm
|
||||||
|
uses: pnpm/action-setup@v4
|
||||||
|
with:
|
||||||
|
version: 11.1.1
|
||||||
|
- name: Setup Node.js 24 for install
|
||||||
|
uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
|
||||||
|
# in October.
|
||||||
|
node-version: 24
|
||||||
|
cache: pnpm
|
||||||
|
cache-dependency-path: nodejs/pnpm-lock.yaml
|
||||||
|
- name: Install dependencies
|
||||||
|
run: pnpm install --frozen-lockfile
|
||||||
|
- name: Setup Node.js ${{ matrix.node }} for test
|
||||||
uses: actions/setup-node@v4
|
uses: actions/setup-node@v4
|
||||||
with:
|
with:
|
||||||
node-version: ${{ matrix.node }}
|
node-version: ${{ matrix.node }}
|
||||||
cache: npm
|
|
||||||
cache-dependency-path: nodejs/package-lock.json
|
|
||||||
- name: Install dependencies
|
|
||||||
run: npm ci
|
|
||||||
- name: Download artifacts
|
- name: Download artifacts
|
||||||
uses: actions/download-artifact@v4
|
uses: actions/download-artifact@v4
|
||||||
with:
|
with:
|
||||||
@@ -311,7 +326,9 @@ jobs:
|
|||||||
- name: Move built files
|
- name: Move built files
|
||||||
run: cp dist/native.d.ts dist/native.js dist/*.node lancedb/
|
run: cp dist/native.d.ts dist/native.js dist/*.node lancedb/
|
||||||
- name: Test bindings
|
- name: Test bindings
|
||||||
run: npm test
|
# Invoke jest directly because pnpm 11 itself requires Node 22+
|
||||||
|
# while the matrix tests on older Node versions.
|
||||||
|
run: npx jest --verbose
|
||||||
publish:
|
publish:
|
||||||
name: Publish
|
name: Publish
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
@@ -323,15 +340,19 @@ jobs:
|
|||||||
- test-lancedb
|
- test-lancedb
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
|
- name: Setup pnpm
|
||||||
|
uses: pnpm/action-setup@v4
|
||||||
|
with:
|
||||||
|
version: 11.1.1
|
||||||
- name: Setup node
|
- name: Setup node
|
||||||
uses: actions/setup-node@v4
|
uses: actions/setup-node@v4
|
||||||
with:
|
with:
|
||||||
node-version: 24
|
node-version: 24
|
||||||
cache: npm
|
cache: pnpm
|
||||||
cache-dependency-path: nodejs/package-lock.json
|
cache-dependency-path: nodejs/pnpm-lock.yaml
|
||||||
registry-url: "https://registry.npmjs.org"
|
registry-url: "https://registry.npmjs.org"
|
||||||
- name: Install dependencies
|
- name: Install dependencies
|
||||||
run: npm ci
|
run: pnpm install --frozen-lockfile
|
||||||
- uses: actions/download-artifact@v4
|
- uses: actions/download-artifact@v4
|
||||||
with:
|
with:
|
||||||
name: nodejs-dist
|
name: nodejs-dist
|
||||||
@@ -351,7 +372,7 @@ jobs:
|
|||||||
- name: Display structure of downloaded files
|
- name: Display structure of downloaded files
|
||||||
run: find dist && find nodejs-artifacts
|
run: find dist && find nodejs-artifacts
|
||||||
- name: Move artifacts
|
- name: Move artifacts
|
||||||
run: npx napi artifacts -d nodejs-artifacts
|
run: pnpm exec napi artifacts -d nodejs-artifacts
|
||||||
- name: List packages
|
- name: List packages
|
||||||
run: find npm
|
run: find npm
|
||||||
- name: Publish
|
- name: Publish
|
||||||
|
|||||||
Generated
+1
@@ -5099,6 +5099,7 @@ dependencies = [
|
|||||||
"env_logger",
|
"env_logger",
|
||||||
"futures",
|
"futures",
|
||||||
"half",
|
"half",
|
||||||
|
"lance-namespace",
|
||||||
"lancedb",
|
"lancedb",
|
||||||
"log",
|
"log",
|
||||||
"lzma-sys",
|
"lzma-sys",
|
||||||
|
|||||||
@@ -12,20 +12,22 @@ Typescript.
|
|||||||
* `src/`: Rust bindings source code
|
* `src/`: Rust bindings source code
|
||||||
* `lancedb/`: Typescript package source code
|
* `lancedb/`: Typescript package source code
|
||||||
* `__test__/`: Unit tests
|
* `__test__/`: Unit tests
|
||||||
* `examples/`: An npm package with the examples shown in the documentation
|
* `examples/`: A pnpm package with the examples shown in the documentation
|
||||||
|
|
||||||
## Development environment
|
## Development environment
|
||||||
|
|
||||||
To set up your development environment, you will need to install the following:
|
To set up your development environment, you will need to install the following:
|
||||||
|
|
||||||
1. Node.js 14 or later
|
1. Node.js 22 or later (required by pnpm 11)
|
||||||
2. Rust's package manager, Cargo. Use [rustup](https://rustup.rs/) to install.
|
2. [pnpm](https://pnpm.io/installation) 11 or later (or run via `corepack enable`,
|
||||||
3. [protoc](https://grpc.io/docs/protoc-installation/) (Protocol Buffers compiler)
|
which uses the `packageManager` field in `package.json`)
|
||||||
|
3. Rust's package manager, Cargo. Use [rustup](https://rustup.rs/) to install.
|
||||||
|
4. [protoc](https://grpc.io/docs/protoc-installation/) (Protocol Buffers compiler)
|
||||||
|
|
||||||
Initial setup:
|
Initial setup:
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
npm install
|
pnpm install
|
||||||
```
|
```
|
||||||
|
|
||||||
### Commit Hooks
|
### Commit Hooks
|
||||||
@@ -39,38 +41,38 @@ pre-commit install
|
|||||||
|
|
||||||
## Development
|
## Development
|
||||||
|
|
||||||
Most common development commands can be run using the npm scripts.
|
Most common development commands can be run using the pnpm scripts.
|
||||||
|
|
||||||
Build the package
|
Build the package
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
npm install
|
pnpm install
|
||||||
npm run build
|
pnpm build
|
||||||
```
|
```
|
||||||
|
|
||||||
Lint:
|
Lint:
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
npm run lint
|
pnpm lint
|
||||||
```
|
```
|
||||||
|
|
||||||
Format and fix lints:
|
Format and fix lints:
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
npm run lint-fix
|
pnpm lint-fix
|
||||||
```
|
```
|
||||||
|
|
||||||
Run tests:
|
Run tests:
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
npm test
|
pnpm test
|
||||||
```
|
```
|
||||||
|
|
||||||
To run a single test:
|
To run a single test:
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
# Single file: table.test.ts
|
# Single file: table.test.ts
|
||||||
npm test -- table.test.ts
|
pnpm test -- table.test.ts
|
||||||
# Single test: 'merge insert' in table.test.ts
|
# Single test: 'merge insert' in table.test.ts
|
||||||
npm test -- table.test.ts --testNamePattern=merge\ insert
|
pnpm test -- table.test.ts --testNamePattern=merge\ insert
|
||||||
```
|
```
|
||||||
|
|||||||
@@ -148,6 +148,33 @@ Creates a new empty Table
|
|||||||
|
|
||||||
***
|
***
|
||||||
|
|
||||||
|
### createNamespace()
|
||||||
|
|
||||||
|
```ts
|
||||||
|
abstract createNamespace(namespacePath, options?): Promise<CreateNamespaceResponse>
|
||||||
|
```
|
||||||
|
|
||||||
|
Create a new namespace at the given path.
|
||||||
|
|
||||||
|
#### Parameters
|
||||||
|
|
||||||
|
* **namespacePath**: `string`[]
|
||||||
|
The namespace path to create.
|
||||||
|
|
||||||
|
* **options?**: `Partial`<[`CreateNamespaceOptions`](../interfaces/CreateNamespaceOptions.md)>
|
||||||
|
Creation `mode`
|
||||||
|
("create" | "exist_ok" | "overwrite") and optional `properties`
|
||||||
|
to attach to the namespace.
|
||||||
|
|
||||||
|
#### Returns
|
||||||
|
|
||||||
|
`Promise`<[`CreateNamespaceResponse`](../interfaces/CreateNamespaceResponse.md)>
|
||||||
|
|
||||||
|
The properties of the
|
||||||
|
created namespace and an optional transaction id.
|
||||||
|
|
||||||
|
***
|
||||||
|
|
||||||
### createTable()
|
### createTable()
|
||||||
|
|
||||||
#### createTable(options, namespacePath)
|
#### createTable(options, namespacePath)
|
||||||
@@ -230,6 +257,29 @@ Creates a new Table and initialize it with new data.
|
|||||||
|
|
||||||
***
|
***
|
||||||
|
|
||||||
|
### describeNamespace()
|
||||||
|
|
||||||
|
```ts
|
||||||
|
abstract describeNamespace(namespacePath): Promise<DescribeNamespaceResponse>
|
||||||
|
```
|
||||||
|
|
||||||
|
Describe a namespace, returning its properties.
|
||||||
|
|
||||||
|
#### Parameters
|
||||||
|
|
||||||
|
* **namespacePath**: `string`[]
|
||||||
|
The namespace path to describe, in
|
||||||
|
parent → child order, e.g. `["analytics", "sales"]`.
|
||||||
|
|
||||||
|
#### Returns
|
||||||
|
|
||||||
|
`Promise`<[`DescribeNamespaceResponse`](../interfaces/DescribeNamespaceResponse.md)>
|
||||||
|
|
||||||
|
The namespace's properties
|
||||||
|
(may be undefined if the namespace has none).
|
||||||
|
|
||||||
|
***
|
||||||
|
|
||||||
### display()
|
### display()
|
||||||
|
|
||||||
```ts
|
```ts
|
||||||
@@ -263,6 +313,36 @@ Drop all tables in the database.
|
|||||||
|
|
||||||
***
|
***
|
||||||
|
|
||||||
|
### dropNamespace()
|
||||||
|
|
||||||
|
```ts
|
||||||
|
abstract dropNamespace(namespacePath, options?): Promise<DropNamespaceResponse>
|
||||||
|
```
|
||||||
|
|
||||||
|
Drop a namespace.
|
||||||
|
|
||||||
|
Use `behavior: "cascade"` to also drop everything contained in the
|
||||||
|
namespace (sub-namespaces and tables). The default `"restrict"`
|
||||||
|
behavior refuses to drop a non-empty namespace.
|
||||||
|
|
||||||
|
#### Parameters
|
||||||
|
|
||||||
|
* **namespacePath**: `string`[]
|
||||||
|
The namespace path to drop.
|
||||||
|
|
||||||
|
* **options?**: `Partial`<[`DropNamespaceOptions`](../interfaces/DropNamespaceOptions.md)>
|
||||||
|
`mode` ("skip" | "fail"
|
||||||
|
for missing-namespace handling) and `behavior` ("restrict" | "cascade").
|
||||||
|
|
||||||
|
#### Returns
|
||||||
|
|
||||||
|
`Promise`<[`DropNamespaceResponse`](../interfaces/DropNamespaceResponse.md)>
|
||||||
|
|
||||||
|
Any properties returned by
|
||||||
|
the server and an optional transaction id.
|
||||||
|
|
||||||
|
***
|
||||||
|
|
||||||
### dropTable()
|
### dropTable()
|
||||||
|
|
||||||
```ts
|
```ts
|
||||||
@@ -299,6 +379,36 @@ Return true if the connection has not been closed
|
|||||||
|
|
||||||
***
|
***
|
||||||
|
|
||||||
|
### listNamespaces()
|
||||||
|
|
||||||
|
```ts
|
||||||
|
abstract listNamespaces(namespacePath?, options?): Promise<ListNamespacesResponse>
|
||||||
|
```
|
||||||
|
|
||||||
|
List the immediate child namespaces under the given parent.
|
||||||
|
|
||||||
|
Results may be paginated. To retrieve subsequent pages, pass the
|
||||||
|
`pageToken` returned by a previous call.
|
||||||
|
|
||||||
|
#### Parameters
|
||||||
|
|
||||||
|
* **namespacePath?**: `string`[]
|
||||||
|
The parent namespace path. Defaults
|
||||||
|
to the root namespace if omitted.
|
||||||
|
|
||||||
|
* **options?**: `Partial`<[`ListNamespacesOptions`](../interfaces/ListNamespacesOptions.md)>
|
||||||
|
Pagination options
|
||||||
|
(`pageToken`, `limit`).
|
||||||
|
|
||||||
|
#### Returns
|
||||||
|
|
||||||
|
`Promise`<[`ListNamespacesResponse`](../interfaces/ListNamespacesResponse.md)>
|
||||||
|
|
||||||
|
Child namespace names and
|
||||||
|
an optional token for fetching the next page.
|
||||||
|
|
||||||
|
***
|
||||||
|
|
||||||
### openTable()
|
### openTable()
|
||||||
|
|
||||||
```ts
|
```ts
|
||||||
|
|||||||
@@ -0,0 +1,131 @@
|
|||||||
|
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||||
|
|
||||||
|
***
|
||||||
|
|
||||||
|
[@lancedb/lancedb](../globals.md) / connectNamespace
|
||||||
|
|
||||||
|
# Function: connectNamespace()
|
||||||
|
|
||||||
|
## connectNamespace(implName, config, options)
|
||||||
|
|
||||||
|
```ts
|
||||||
|
function connectNamespace(
|
||||||
|
implName,
|
||||||
|
config,
|
||||||
|
options?): Promise<Connection>
|
||||||
|
```
|
||||||
|
|
||||||
|
Connect to a LanceDB database through a namespace.
|
||||||
|
|
||||||
|
Unlike [connect](connect.md), which routes by URI scheme (local path vs.
|
||||||
|
`db://` cloud), `connectNamespace` always returns a namespace-backed
|
||||||
|
connection. The `implName` selects the namespace implementation:
|
||||||
|
|
||||||
|
- `"dir"` — directory namespace, configured with [DirNamespaceConfig](../interfaces/DirNamespaceConfig.md).
|
||||||
|
- `"rest"` — remote REST catalog, configured with [RestNamespaceConfig](../interfaces/RestNamespaceConfig.md).
|
||||||
|
- Any other string — full module path for a custom implementation,
|
||||||
|
configured with a free-form string-keyed `properties` map.
|
||||||
|
|
||||||
|
### Parameters
|
||||||
|
|
||||||
|
* **implName**: `"dir"`
|
||||||
|
|
||||||
|
* **config**: [`DirNamespaceConfig`](../interfaces/DirNamespaceConfig.md)
|
||||||
|
|
||||||
|
* **options?**: `Partial`<[`ConnectNamespaceOptions`](../interfaces/ConnectNamespaceOptions.md)>
|
||||||
|
|
||||||
|
### Returns
|
||||||
|
|
||||||
|
`Promise`<[`Connection`](../classes/Connection.md)>
|
||||||
|
|
||||||
|
### Examples
|
||||||
|
|
||||||
|
```ts
|
||||||
|
const db = await connectNamespace("dir", { root: "/path/to/db" });
|
||||||
|
await db.createTable("users", [{ id: 1 }]);
|
||||||
|
```
|
||||||
|
|
||||||
|
```ts
|
||||||
|
const db = await connectNamespace("rest", {
|
||||||
|
uri: "https://catalog.example.com",
|
||||||
|
headers: { "x-api-key": process.env.CATALOG_KEY ?? "" },
|
||||||
|
});
|
||||||
|
```
|
||||||
|
|
||||||
|
```ts
|
||||||
|
const db = await connectNamespace("my.custom.Namespace", {
|
||||||
|
endpoint: "...",
|
||||||
|
});
|
||||||
|
```
|
||||||
|
|
||||||
|
## connectNamespace(implName, config, options)
|
||||||
|
|
||||||
|
```ts
|
||||||
|
function connectNamespace(
|
||||||
|
implName,
|
||||||
|
config,
|
||||||
|
options?): Promise<Connection>
|
||||||
|
```
|
||||||
|
|
||||||
|
Connect through the built-in REST namespace.
|
||||||
|
|
||||||
|
Configured with [RestNamespaceConfig](../interfaces/RestNamespaceConfig.md). See the function-level
|
||||||
|
documentation above for the full surface, examples, and how this
|
||||||
|
relates to [connect](connect.md).
|
||||||
|
|
||||||
|
### Parameters
|
||||||
|
|
||||||
|
* **implName**: `"rest"`
|
||||||
|
|
||||||
|
* **config**: [`RestNamespaceConfig`](../interfaces/RestNamespaceConfig.md)
|
||||||
|
|
||||||
|
* **options?**: `Partial`<[`ConnectNamespaceOptions`](../interfaces/ConnectNamespaceOptions.md)>
|
||||||
|
|
||||||
|
### Returns
|
||||||
|
|
||||||
|
`Promise`<[`Connection`](../classes/Connection.md)>
|
||||||
|
|
||||||
|
### Example
|
||||||
|
|
||||||
|
```ts
|
||||||
|
const db = await connectNamespace("rest", {
|
||||||
|
uri: "https://catalog.example.com",
|
||||||
|
headers: { "x-api-key": process.env.CATALOG_KEY ?? "" },
|
||||||
|
});
|
||||||
|
```
|
||||||
|
|
||||||
|
## connectNamespace(implName, properties, options)
|
||||||
|
|
||||||
|
```ts
|
||||||
|
function connectNamespace(
|
||||||
|
implName,
|
||||||
|
properties,
|
||||||
|
options?): Promise<Connection>
|
||||||
|
```
|
||||||
|
|
||||||
|
Connect through a custom namespace implementation by full module path,
|
||||||
|
configured with a free-form string-keyed `properties` map. Use the
|
||||||
|
typed overloads above for the built-in `"dir"` and `"rest"` impls.
|
||||||
|
|
||||||
|
See the function-level documentation above for examples and how this
|
||||||
|
relates to [connect](connect.md).
|
||||||
|
|
||||||
|
### Parameters
|
||||||
|
|
||||||
|
* **implName**: `string`
|
||||||
|
|
||||||
|
* **properties**: `Record`<`string`, `string`>
|
||||||
|
|
||||||
|
* **options?**: `Partial`<[`ConnectNamespaceOptions`](../interfaces/ConnectNamespaceOptions.md)>
|
||||||
|
|
||||||
|
### Returns
|
||||||
|
|
||||||
|
`Promise`<[`Connection`](../classes/Connection.md)>
|
||||||
|
|
||||||
|
### Example
|
||||||
|
|
||||||
|
```ts
|
||||||
|
const db = await connectNamespace("my.custom.Namespace", {
|
||||||
|
endpoint: "...",
|
||||||
|
});
|
||||||
|
```
|
||||||
@@ -51,10 +51,17 @@
|
|||||||
- [ClientConfig](interfaces/ClientConfig.md)
|
- [ClientConfig](interfaces/ClientConfig.md)
|
||||||
- [ColumnAlteration](interfaces/ColumnAlteration.md)
|
- [ColumnAlteration](interfaces/ColumnAlteration.md)
|
||||||
- [CompactionStats](interfaces/CompactionStats.md)
|
- [CompactionStats](interfaces/CompactionStats.md)
|
||||||
|
- [ConnectNamespaceOptions](interfaces/ConnectNamespaceOptions.md)
|
||||||
- [ConnectionOptions](interfaces/ConnectionOptions.md)
|
- [ConnectionOptions](interfaces/ConnectionOptions.md)
|
||||||
|
- [CreateNamespaceOptions](interfaces/CreateNamespaceOptions.md)
|
||||||
|
- [CreateNamespaceResponse](interfaces/CreateNamespaceResponse.md)
|
||||||
- [CreateTableOptions](interfaces/CreateTableOptions.md)
|
- [CreateTableOptions](interfaces/CreateTableOptions.md)
|
||||||
- [DeleteResult](interfaces/DeleteResult.md)
|
- [DeleteResult](interfaces/DeleteResult.md)
|
||||||
|
- [DescribeNamespaceResponse](interfaces/DescribeNamespaceResponse.md)
|
||||||
|
- [DirNamespaceConfig](interfaces/DirNamespaceConfig.md)
|
||||||
- [DropColumnsResult](interfaces/DropColumnsResult.md)
|
- [DropColumnsResult](interfaces/DropColumnsResult.md)
|
||||||
|
- [DropNamespaceOptions](interfaces/DropNamespaceOptions.md)
|
||||||
|
- [DropNamespaceResponse](interfaces/DropNamespaceResponse.md)
|
||||||
- [ExecutableQuery](interfaces/ExecutableQuery.md)
|
- [ExecutableQuery](interfaces/ExecutableQuery.md)
|
||||||
- [FragmentStatistics](interfaces/FragmentStatistics.md)
|
- [FragmentStatistics](interfaces/FragmentStatistics.md)
|
||||||
- [FragmentSummaryStats](interfaces/FragmentSummaryStats.md)
|
- [FragmentSummaryStats](interfaces/FragmentSummaryStats.md)
|
||||||
@@ -69,12 +76,15 @@
|
|||||||
- [IvfFlatOptions](interfaces/IvfFlatOptions.md)
|
- [IvfFlatOptions](interfaces/IvfFlatOptions.md)
|
||||||
- [IvfPqOptions](interfaces/IvfPqOptions.md)
|
- [IvfPqOptions](interfaces/IvfPqOptions.md)
|
||||||
- [IvfRqOptions](interfaces/IvfRqOptions.md)
|
- [IvfRqOptions](interfaces/IvfRqOptions.md)
|
||||||
|
- [ListNamespacesOptions](interfaces/ListNamespacesOptions.md)
|
||||||
|
- [ListNamespacesResponse](interfaces/ListNamespacesResponse.md)
|
||||||
- [MergeResult](interfaces/MergeResult.md)
|
- [MergeResult](interfaces/MergeResult.md)
|
||||||
- [OpenTableOptions](interfaces/OpenTableOptions.md)
|
- [OpenTableOptions](interfaces/OpenTableOptions.md)
|
||||||
- [OptimizeOptions](interfaces/OptimizeOptions.md)
|
- [OptimizeOptions](interfaces/OptimizeOptions.md)
|
||||||
- [OptimizeStats](interfaces/OptimizeStats.md)
|
- [OptimizeStats](interfaces/OptimizeStats.md)
|
||||||
- [QueryExecutionOptions](interfaces/QueryExecutionOptions.md)
|
- [QueryExecutionOptions](interfaces/QueryExecutionOptions.md)
|
||||||
- [RemovalStats](interfaces/RemovalStats.md)
|
- [RemovalStats](interfaces/RemovalStats.md)
|
||||||
|
- [RestNamespaceConfig](interfaces/RestNamespaceConfig.md)
|
||||||
- [RetryConfig](interfaces/RetryConfig.md)
|
- [RetryConfig](interfaces/RetryConfig.md)
|
||||||
- [ShuffleOptions](interfaces/ShuffleOptions.md)
|
- [ShuffleOptions](interfaces/ShuffleOptions.md)
|
||||||
- [SplitCalculatedOptions](interfaces/SplitCalculatedOptions.md)
|
- [SplitCalculatedOptions](interfaces/SplitCalculatedOptions.md)
|
||||||
@@ -107,6 +117,7 @@
|
|||||||
|
|
||||||
- [RecordBatchIterator](functions/RecordBatchIterator.md)
|
- [RecordBatchIterator](functions/RecordBatchIterator.md)
|
||||||
- [connect](functions/connect.md)
|
- [connect](functions/connect.md)
|
||||||
|
- [connectNamespace](functions/connectNamespace.md)
|
||||||
- [makeArrowTable](functions/makeArrowTable.md)
|
- [makeArrowTable](functions/makeArrowTable.md)
|
||||||
- [packBits](functions/packBits.md)
|
- [packBits](functions/packBits.md)
|
||||||
- [permutationBuilder](functions/permutationBuilder.md)
|
- [permutationBuilder](functions/permutationBuilder.md)
|
||||||
|
|||||||
@@ -0,0 +1,54 @@
|
|||||||
|
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||||
|
|
||||||
|
***
|
||||||
|
|
||||||
|
[@lancedb/lancedb](../globals.md) / ConnectNamespaceOptions
|
||||||
|
|
||||||
|
# Interface: ConnectNamespaceOptions
|
||||||
|
|
||||||
|
## Properties
|
||||||
|
|
||||||
|
### namespaceClientProperties?
|
||||||
|
|
||||||
|
```ts
|
||||||
|
optional namespaceClientProperties: Record<string, string>;
|
||||||
|
```
|
||||||
|
|
||||||
|
Extra properties for the backing namespace client.
|
||||||
|
|
||||||
|
***
|
||||||
|
|
||||||
|
### readConsistencyInterval?
|
||||||
|
|
||||||
|
```ts
|
||||||
|
optional readConsistencyInterval: number;
|
||||||
|
```
|
||||||
|
|
||||||
|
The interval, in seconds, at which to check for updates to the table
|
||||||
|
from other processes. If None, then consistency is not checked. For
|
||||||
|
performance reasons, this is the default. For strong consistency, set
|
||||||
|
this to zero seconds. Then every read will check for updates from other
|
||||||
|
processes. As a compromise, you can set this to a non-zero value for
|
||||||
|
eventual consistency.
|
||||||
|
|
||||||
|
***
|
||||||
|
|
||||||
|
### session?
|
||||||
|
|
||||||
|
```ts
|
||||||
|
optional session: Session;
|
||||||
|
```
|
||||||
|
|
||||||
|
The session to use for this connection. Holds shared caches and other
|
||||||
|
session-specific state.
|
||||||
|
|
||||||
|
***
|
||||||
|
|
||||||
|
### storageOptions?
|
||||||
|
|
||||||
|
```ts
|
||||||
|
optional storageOptions: Record<string, string>;
|
||||||
|
```
|
||||||
|
|
||||||
|
Configuration for object storage. The available options are described
|
||||||
|
at https://docs.lancedb.com/storage/
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||||
|
|
||||||
|
***
|
||||||
|
|
||||||
|
[@lancedb/lancedb](../globals.md) / CreateNamespaceOptions
|
||||||
|
|
||||||
|
# Interface: CreateNamespaceOptions
|
||||||
|
|
||||||
|
## Properties
|
||||||
|
|
||||||
|
### mode?
|
||||||
|
|
||||||
|
```ts
|
||||||
|
optional mode: "overwrite" | "create" | "exist_ok";
|
||||||
|
```
|
||||||
|
|
||||||
|
Creation mode.
|
||||||
|
|
||||||
|
***
|
||||||
|
|
||||||
|
### properties?
|
||||||
|
|
||||||
|
```ts
|
||||||
|
optional properties: Record<string, string>;
|
||||||
|
```
|
||||||
|
|
||||||
|
Properties to set on the new namespace.
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||||
|
|
||||||
|
***
|
||||||
|
|
||||||
|
[@lancedb/lancedb](../globals.md) / CreateNamespaceResponse
|
||||||
|
|
||||||
|
# Interface: CreateNamespaceResponse
|
||||||
|
|
||||||
|
## Properties
|
||||||
|
|
||||||
|
### properties?
|
||||||
|
|
||||||
|
```ts
|
||||||
|
optional properties: Record<string, string>;
|
||||||
|
```
|
||||||
|
|
||||||
|
***
|
||||||
|
|
||||||
|
### transactionId?
|
||||||
|
|
||||||
|
```ts
|
||||||
|
optional transactionId: string;
|
||||||
|
```
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||||
|
|
||||||
|
***
|
||||||
|
|
||||||
|
[@lancedb/lancedb](../globals.md) / DescribeNamespaceResponse
|
||||||
|
|
||||||
|
# Interface: DescribeNamespaceResponse
|
||||||
|
|
||||||
|
## Properties
|
||||||
|
|
||||||
|
### properties?
|
||||||
|
|
||||||
|
```ts
|
||||||
|
optional properties: Record<string, string>;
|
||||||
|
```
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||||
|
|
||||||
|
***
|
||||||
|
|
||||||
|
[@lancedb/lancedb](../globals.md) / DirNamespaceConfig
|
||||||
|
|
||||||
|
# Interface: DirNamespaceConfig
|
||||||
|
|
||||||
|
Configuration for the built-in directory namespace (`"dir"`).
|
||||||
|
|
||||||
|
The directory namespace stores tables under a single root path (local
|
||||||
|
filesystem or object storage URI). See
|
||||||
|
[https://docs.lancedb.com/namespaces](https://docs.lancedb.com/namespaces) for the documented surface;
|
||||||
|
less-common knobs live under [DirNamespaceConfig.extraProperties](DirNamespaceConfig.md#extraproperties).
|
||||||
|
|
||||||
|
## Properties
|
||||||
|
|
||||||
|
### extraProperties?
|
||||||
|
|
||||||
|
```ts
|
||||||
|
optional extraProperties: Record<string, string>;
|
||||||
|
```
|
||||||
|
|
||||||
|
Additional raw properties passed verbatim to the namespace
|
||||||
|
implementation (e.g. `storage.*`, `credential_vendor.*`). Typed
|
||||||
|
fields above take precedence on key collision.
|
||||||
|
|
||||||
|
***
|
||||||
|
|
||||||
|
### manifestEnabled?
|
||||||
|
|
||||||
|
```ts
|
||||||
|
optional manifestEnabled: boolean;
|
||||||
|
```
|
||||||
|
|
||||||
|
Whether to maintain a namespace manifest at the root. Required for
|
||||||
|
child namespaces. Defaults to true on the impl side.
|
||||||
|
|
||||||
|
***
|
||||||
|
|
||||||
|
### root
|
||||||
|
|
||||||
|
```ts
|
||||||
|
root: string;
|
||||||
|
```
|
||||||
|
|
||||||
|
Root path or URI containing the LanceDB tables.
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||||
|
|
||||||
|
***
|
||||||
|
|
||||||
|
[@lancedb/lancedb](../globals.md) / DropNamespaceOptions
|
||||||
|
|
||||||
|
# Interface: DropNamespaceOptions
|
||||||
|
|
||||||
|
## Properties
|
||||||
|
|
||||||
|
### behavior?
|
||||||
|
|
||||||
|
```ts
|
||||||
|
optional behavior: "restrict" | "cascade";
|
||||||
|
```
|
||||||
|
|
||||||
|
Refuse to drop if non-empty (restrict) or drop recursively (cascade).
|
||||||
|
|
||||||
|
***
|
||||||
|
|
||||||
|
### mode?
|
||||||
|
|
||||||
|
```ts
|
||||||
|
optional mode: "fail" | "skip";
|
||||||
|
```
|
||||||
|
|
||||||
|
Whether to skip if the namespace doesn't exist, or fail.
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||||
|
|
||||||
|
***
|
||||||
|
|
||||||
|
[@lancedb/lancedb](../globals.md) / DropNamespaceResponse
|
||||||
|
|
||||||
|
# Interface: DropNamespaceResponse
|
||||||
|
|
||||||
|
## Properties
|
||||||
|
|
||||||
|
### properties?
|
||||||
|
|
||||||
|
```ts
|
||||||
|
optional properties: Record<string, string>;
|
||||||
|
```
|
||||||
|
|
||||||
|
***
|
||||||
|
|
||||||
|
### transactionId?
|
||||||
|
|
||||||
|
```ts
|
||||||
|
optional transactionId: string[];
|
||||||
|
```
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||||
|
|
||||||
|
***
|
||||||
|
|
||||||
|
[@lancedb/lancedb](../globals.md) / ListNamespacesOptions
|
||||||
|
|
||||||
|
# Interface: ListNamespacesOptions
|
||||||
|
|
||||||
|
## Properties
|
||||||
|
|
||||||
|
### limit?
|
||||||
|
|
||||||
|
```ts
|
||||||
|
optional limit: number;
|
||||||
|
```
|
||||||
|
|
||||||
|
An optional limit to the number of results to return.
|
||||||
|
|
||||||
|
***
|
||||||
|
|
||||||
|
### pageToken?
|
||||||
|
|
||||||
|
```ts
|
||||||
|
optional pageToken: string;
|
||||||
|
```
|
||||||
|
|
||||||
|
Token from a previous response for pagination.
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||||
|
|
||||||
|
***
|
||||||
|
|
||||||
|
[@lancedb/lancedb](../globals.md) / ListNamespacesResponse
|
||||||
|
|
||||||
|
# Interface: ListNamespacesResponse
|
||||||
|
|
||||||
|
## Properties
|
||||||
|
|
||||||
|
### namespaces
|
||||||
|
|
||||||
|
```ts
|
||||||
|
namespaces: string[];
|
||||||
|
```
|
||||||
|
|
||||||
|
***
|
||||||
|
|
||||||
|
### pageToken?
|
||||||
|
|
||||||
|
```ts
|
||||||
|
optional pageToken: string;
|
||||||
|
```
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||||
|
|
||||||
|
***
|
||||||
|
|
||||||
|
[@lancedb/lancedb](../globals.md) / RestNamespaceConfig
|
||||||
|
|
||||||
|
# Interface: RestNamespaceConfig
|
||||||
|
|
||||||
|
Configuration for the built-in REST namespace (`"rest"`).
|
||||||
|
|
||||||
|
The REST namespace talks to a remote catalog server over HTTP. See
|
||||||
|
[https://docs.lancedb.com/namespaces](https://docs.lancedb.com/namespaces) for the documented surface;
|
||||||
|
less-common knobs (TLS, metrics) live under
|
||||||
|
[RestNamespaceConfig.extraProperties](RestNamespaceConfig.md#extraproperties).
|
||||||
|
|
||||||
|
## Properties
|
||||||
|
|
||||||
|
### extraProperties?
|
||||||
|
|
||||||
|
```ts
|
||||||
|
optional extraProperties: Record<string, string>;
|
||||||
|
```
|
||||||
|
|
||||||
|
Additional raw properties passed verbatim to the namespace
|
||||||
|
implementation (e.g. `tls.*`, `ops_metrics_enabled`, `delimiter`).
|
||||||
|
Typed fields above take precedence on key collision.
|
||||||
|
|
||||||
|
***
|
||||||
|
|
||||||
|
### headers?
|
||||||
|
|
||||||
|
```ts
|
||||||
|
optional headers: Record<string, string>;
|
||||||
|
```
|
||||||
|
|
||||||
|
HTTP headers forwarded with each request. Keys are passed through
|
||||||
|
as-is (e.g. `"x-api-key"`, `"Authorization"`).
|
||||||
|
|
||||||
|
***
|
||||||
|
|
||||||
|
### uri
|
||||||
|
|
||||||
|
```ts
|
||||||
|
uri: string;
|
||||||
|
```
|
||||||
|
|
||||||
|
Catalog endpoint URL.
|
||||||
+6
-6
@@ -3,11 +3,11 @@ The core Rust library is in the `../rust/lancedb` directory, the rust binding
|
|||||||
code is in the `src/` directory and the typescript bindings are in
|
code is in the `src/` directory and the typescript bindings are in
|
||||||
the `lancedb/` directory.
|
the `lancedb/` directory.
|
||||||
|
|
||||||
Whenever you change the Rust code, you will need to recompile: `npm run build`.
|
Whenever you change the Rust code, you will need to recompile: `pnpm build`.
|
||||||
|
|
||||||
Common commands:
|
Common commands:
|
||||||
* Build: `npm run build`
|
* Build: `pnpm build`
|
||||||
* Lint: `npm run lint`
|
* Lint: `pnpm lint`
|
||||||
* Fix lints: `npm run lint-fix`
|
* Fix lints: `pnpm lint-fix`
|
||||||
* Test: `npm test`
|
* Test: `pnpm test`
|
||||||
* Run single test file: `npm test __test__/arrow.test.ts`
|
* Run single test file: `pnpm test __test__/arrow.test.ts`
|
||||||
|
|||||||
+15
-13
@@ -12,20 +12,22 @@ Typescript.
|
|||||||
* `src/`: Rust bindings source code
|
* `src/`: Rust bindings source code
|
||||||
* `lancedb/`: Typescript package source code
|
* `lancedb/`: Typescript package source code
|
||||||
* `__test__/`: Unit tests
|
* `__test__/`: Unit tests
|
||||||
* `examples/`: An npm package with the examples shown in the documentation
|
* `examples/`: A pnpm package with the examples shown in the documentation
|
||||||
|
|
||||||
## Development environment
|
## Development environment
|
||||||
|
|
||||||
To set up your development environment, you will need to install the following:
|
To set up your development environment, you will need to install the following:
|
||||||
|
|
||||||
1. Node.js 14 or later
|
1. Node.js 22 or later (required by pnpm 11)
|
||||||
2. Rust's package manager, Cargo. Use [rustup](https://rustup.rs/) to install.
|
2. [pnpm](https://pnpm.io/installation) 11 or later (or run via `corepack enable`,
|
||||||
3. [protoc](https://grpc.io/docs/protoc-installation/) (Protocol Buffers compiler)
|
which uses the `packageManager` field in `package.json`)
|
||||||
|
3. Rust's package manager, Cargo. Use [rustup](https://rustup.rs/) to install.
|
||||||
|
4. [protoc](https://grpc.io/docs/protoc-installation/) (Protocol Buffers compiler)
|
||||||
|
|
||||||
Initial setup:
|
Initial setup:
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
npm install
|
pnpm install
|
||||||
```
|
```
|
||||||
|
|
||||||
### Commit Hooks
|
### Commit Hooks
|
||||||
@@ -39,38 +41,38 @@ pre-commit install
|
|||||||
|
|
||||||
## Development
|
## Development
|
||||||
|
|
||||||
Most common development commands can be run using the npm scripts.
|
Most common development commands can be run using the pnpm scripts.
|
||||||
|
|
||||||
Build the package
|
Build the package
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
npm install
|
pnpm install
|
||||||
npm run build
|
pnpm build
|
||||||
```
|
```
|
||||||
|
|
||||||
Lint:
|
Lint:
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
npm run lint
|
pnpm lint
|
||||||
```
|
```
|
||||||
|
|
||||||
Format and fix lints:
|
Format and fix lints:
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
npm run lint-fix
|
pnpm lint-fix
|
||||||
```
|
```
|
||||||
|
|
||||||
Run tests:
|
Run tests:
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
npm test
|
pnpm test
|
||||||
```
|
```
|
||||||
|
|
||||||
To run a single test:
|
To run a single test:
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
# Single file: table.test.ts
|
# Single file: table.test.ts
|
||||||
npm test -- table.test.ts
|
pnpm test -- table.test.ts
|
||||||
# Single test: 'merge insert' in table.test.ts
|
# Single test: 'merge insert' in table.test.ts
|
||||||
npm test -- table.test.ts --testNamePattern=merge\ insert
|
pnpm test -- table.test.ts --testNamePattern=merge\ insert
|
||||||
```
|
```
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ arrow-schema.workspace = true
|
|||||||
env_logger.workspace = true
|
env_logger.workspace = true
|
||||||
futures.workspace = true
|
futures.workspace = true
|
||||||
lancedb = { path = "../rust/lancedb", default-features = false }
|
lancedb = { path = "../rust/lancedb", default-features = false }
|
||||||
|
lance-namespace.workspace = true
|
||||||
napi = { version = "3.8.3", default-features = false, features = [
|
napi = { version = "3.8.3", default-features = false, features = [
|
||||||
"napi9",
|
"napi9",
|
||||||
"async"
|
"async"
|
||||||
|
|||||||
@@ -4,7 +4,7 @@
|
|||||||
import { readdirSync } from "fs";
|
import { readdirSync } from "fs";
|
||||||
import { Field, Float64, Schema } from "apache-arrow";
|
import { Field, Float64, Schema } from "apache-arrow";
|
||||||
import * as tmp from "tmp";
|
import * as tmp from "tmp";
|
||||||
import { Connection, Table, connect } from "../lancedb";
|
import { Connection, Table, connect, connectNamespace } from "../lancedb";
|
||||||
import { LocalTable } from "../lancedb/table";
|
import { LocalTable } from "../lancedb/table";
|
||||||
|
|
||||||
describe("when connecting", () => {
|
describe("when connecting", () => {
|
||||||
@@ -306,3 +306,186 @@ describe("clone table functionality", () => {
|
|||||||
).rejects.toThrow("Deep clone is not yet implemented");
|
).rejects.toThrow("Deep clone is not yet implemented");
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("namespaces", () => {
|
||||||
|
let tmpDir: tmp.DirResult;
|
||||||
|
let db: Connection;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
tmpDir = tmp.dirSync({ unsafeCleanup: true });
|
||||||
|
// The local DirectoryNamespace backend only supports child namespaces
|
||||||
|
// when manifest mode is enabled (see lance-namespace-impls/src/dir.rs).
|
||||||
|
db = await connect(tmpDir.name, {
|
||||||
|
// biome-ignore lint/style/useNamingConvention: opaque backend property key, must match Rust
|
||||||
|
namespaceClientProperties: { manifest_enabled: "true" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
afterEach(() => tmpDir.removeCallback());
|
||||||
|
|
||||||
|
it("should create and describe a namespace", async () => {
|
||||||
|
await db.createNamespace(["myns"]);
|
||||||
|
const desc = await db.describeNamespace(["myns"]);
|
||||||
|
expect(desc).toBeDefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should list namespaces created at the root", async () => {
|
||||||
|
await db.createNamespace(["alpha"]);
|
||||||
|
await db.createNamespace(["beta"]);
|
||||||
|
const list = await db.listNamespaces();
|
||||||
|
expect(list.namespaces).toEqual(expect.arrayContaining(["alpha", "beta"]));
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should list child namespaces under a parent", async () => {
|
||||||
|
await db.createNamespace(["parent"]);
|
||||||
|
await db.createNamespace(["parent", "child"]);
|
||||||
|
const list = await db.listNamespaces(["parent"]);
|
||||||
|
expect(list.namespaces).toContain("child");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should drop a namespace", async () => {
|
||||||
|
await db.createNamespace(["ephemeral"]);
|
||||||
|
await db.dropNamespace(["ephemeral"]);
|
||||||
|
const list = await db.listNamespaces();
|
||||||
|
expect(list.namespaces).not.toContain("ephemeral");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should raise an error on any namespace op after close", async () => {
|
||||||
|
await db.close();
|
||||||
|
await expect(db.describeNamespace(["foo"])).rejects.toThrow(
|
||||||
|
"Connection is closed",
|
||||||
|
);
|
||||||
|
await expect(db.listNamespaces()).rejects.toThrow("Connection is closed");
|
||||||
|
await expect(db.createNamespace(["foo"])).rejects.toThrow(
|
||||||
|
"Connection is closed",
|
||||||
|
);
|
||||||
|
await expect(db.dropNamespace(["foo"])).rejects.toThrow(
|
||||||
|
"Connection is closed",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should raise an understandable error when describing a non-existent namespace", async () => {
|
||||||
|
await expect(db.describeNamespace(["does-not-exist"])).rejects.toThrow(
|
||||||
|
/not found/i,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should raise an error when creating a namespace that already exists", async () => {
|
||||||
|
await db.createNamespace(["dup"]);
|
||||||
|
await expect(db.createNamespace(["dup"])).rejects.toThrow();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should reject an unrecognized createNamespace mode with a clear error", async () => {
|
||||||
|
await expect(
|
||||||
|
// biome-ignore lint/suspicious/noExplicitAny: deliberately bypass TS to test runtime validation
|
||||||
|
db.createNamespace(["x"], { mode: "frobnicate" as any }),
|
||||||
|
).rejects.toThrow(/Invalid mode 'frobnicate'/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should reject an unrecognized dropNamespace mode with a clear error", async () => {
|
||||||
|
await db.createNamespace(["x"]);
|
||||||
|
await expect(
|
||||||
|
// biome-ignore lint/suspicious/noExplicitAny: deliberately bypass TS to test runtime validation
|
||||||
|
db.dropNamespace(["x"], { mode: "frobnicate" as any }),
|
||||||
|
).rejects.toThrow(/Invalid mode 'frobnicate'/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should reject an unrecognized dropNamespace behavior with a clear error", async () => {
|
||||||
|
await db.createNamespace(["x"]);
|
||||||
|
await expect(
|
||||||
|
// biome-ignore lint/suspicious/noExplicitAny: deliberately bypass TS to test runtime validation
|
||||||
|
db.dropNamespace(["x"], { behavior: "frobnicate" as any }),
|
||||||
|
).rejects.toThrow(/Invalid behavior 'frobnicate'/);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("connectNamespace", () => {
|
||||||
|
let tmpDir: tmp.DirResult;
|
||||||
|
beforeEach(() => {
|
||||||
|
tmpDir = tmp.dirSync({ unsafeCleanup: true });
|
||||||
|
});
|
||||||
|
afterEach(() => tmpDir.removeCallback());
|
||||||
|
|
||||||
|
it("connects via the dir implementation and supports table ops", async () => {
|
||||||
|
const db = await connectNamespace("dir", { root: tmpDir.name });
|
||||||
|
await db.createTable("users", [{ id: 1 }, { id: 2 }]);
|
||||||
|
await expect(db.tableNames()).resolves.toContain("users");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("throws a clear error when implName is empty", async () => {
|
||||||
|
await expect(connectNamespace("", {})).rejects.toThrow(
|
||||||
|
"implName must be a non-empty string",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("throws when the namespace implementation is unknown", async () => {
|
||||||
|
await expect(connectNamespace("not-a-real-impl", {})).rejects.toThrow();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("passes storage options through to the namespace", async () => {
|
||||||
|
const db = await connectNamespace(
|
||||||
|
"dir",
|
||||||
|
{ root: tmpDir.name },
|
||||||
|
{ storageOptions: { newTableDataStorageVersion: "stable" } },
|
||||||
|
);
|
||||||
|
await db.createTable("plumbing", [{ id: 1 }]);
|
||||||
|
await expect(db.tableNames()).resolves.toContain("plumbing");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("supports child namespaces when manifestEnabled is true on the dir config", async () => {
|
||||||
|
const writer = await connectNamespace("dir", {
|
||||||
|
root: tmpDir.name,
|
||||||
|
manifestEnabled: true,
|
||||||
|
});
|
||||||
|
await writer.createNamespace(["analytics"]);
|
||||||
|
await writer.createTable("orders", [{ id: 1 }, { id: 2 }], ["analytics"]);
|
||||||
|
await writer.close();
|
||||||
|
|
||||||
|
const reader = await connectNamespace("dir", {
|
||||||
|
root: tmpDir.name,
|
||||||
|
manifestEnabled: true,
|
||||||
|
});
|
||||||
|
await expect(reader.tableNames(["analytics"])).resolves.toContain("orders");
|
||||||
|
const orders = await reader.openTable("orders", ["analytics"]);
|
||||||
|
await expect(orders.countRows()).resolves.toBe(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("merges extraProperties into the dir config and is overridden by typed fields", async () => {
|
||||||
|
// Two observable assertions:
|
||||||
|
// - Typed `root` overrides extraProperties.root: createTable would fail
|
||||||
|
// under the bogus path if the override didn't happen.
|
||||||
|
// - extraProperties.manifest_enabled="false" is honored end-to-end. Child
|
||||||
|
// namespaces require manifest mode (default true), so explicitly
|
||||||
|
// disabling it via extraProperties must make createNamespace reject. If
|
||||||
|
// extraProperties pass-through were silently broken, the default would
|
||||||
|
// let createNamespace succeed.
|
||||||
|
const db = await connectNamespace("dir", {
|
||||||
|
root: tmpDir.name,
|
||||||
|
extraProperties: {
|
||||||
|
root: "/should/be/overridden",
|
||||||
|
// biome-ignore lint/style/useNamingConvention: backend property key
|
||||||
|
manifest_enabled: "false",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await db.createTable("base", [{ id: 1 }]);
|
||||||
|
await expect(db.tableNames()).resolves.toContain("base");
|
||||||
|
await expect(db.createNamespace(["analytics"])).rejects.toThrow();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("flows unknown top-level keys through when implName is dynamic (no silent drop)", async () => {
|
||||||
|
// Routes via the third overload because `impl` is `string`, not the
|
||||||
|
// literal `"dir"`. The dispatcher still notices the runtime value is
|
||||||
|
// "dir", but unknown keys like `manifest_enabled` must not be silently
|
||||||
|
// dropped during the conversion.
|
||||||
|
//
|
||||||
|
// Asserting a *negative* outcome (manifest disabled -> createNamespace
|
||||||
|
// rejects) is required for observability, since the backend default for
|
||||||
|
// `manifest_enabled` is true.
|
||||||
|
const impl: string = "dir";
|
||||||
|
const db = await connectNamespace(impl, {
|
||||||
|
root: tmpDir.name,
|
||||||
|
// biome-ignore lint/style/useNamingConvention: backend property key
|
||||||
|
manifest_enabled: "false",
|
||||||
|
});
|
||||||
|
await expect(db.createNamespace(["mixed"])).rejects.toThrow();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -115,6 +115,12 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
|||||||
await expect(table.countRows()).resolves.toBe(1);
|
await expect(table.countRows()).resolves.toBe(1);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("should accept skipAutoCleanup on add()", async () => {
|
||||||
|
await table.add([{ id: 1 }], { skipAutoCleanup: true });
|
||||||
|
await table.add([{ id: 2 }], { skipAutoCleanup: true });
|
||||||
|
await expect(table.countRows()).resolves.toBe(2);
|
||||||
|
});
|
||||||
|
|
||||||
it("should let me close the table", async () => {
|
it("should let me close the table", async () => {
|
||||||
expect(table.isOpen()).toBe(true);
|
expect(table.isOpen()).toBe(true);
|
||||||
table.close();
|
table.close();
|
||||||
|
|||||||
Generated
-4810
File diff suppressed because it is too large
Load Diff
@@ -11,16 +11,17 @@
|
|||||||
"test": "node --experimental-vm-modules node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose",
|
"test": "node --experimental-vm-modules node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose",
|
||||||
"lint": "biome check *.ts && biome format *.ts",
|
"lint": "biome check *.ts && biome format *.ts",
|
||||||
"lint-ci": "biome ci .",
|
"lint-ci": "biome ci .",
|
||||||
"lint-fix": "biome check --write *.ts && npm run format",
|
"lint-fix": "biome check --write *.ts && pnpm format",
|
||||||
"format": "biome format --write *.ts"
|
"format": "biome format --write *.ts"
|
||||||
},
|
},
|
||||||
"author": "Lance Devs",
|
"author": "Lance Devs",
|
||||||
"license": "Apache-2.0",
|
"license": "Apache-2.0",
|
||||||
|
"packageManager": "pnpm@11.1.1",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@huggingface/transformers": "^3.0.2",
|
"@huggingface/transformers": "3.0.2",
|
||||||
"@lancedb/lancedb": "file:../dist",
|
"@lancedb/lancedb": "file:../dist",
|
||||||
"openai": "^4.29.2",
|
"openai": "4.29.2",
|
||||||
"sharp": "^0.33.5"
|
"sharp": "0.33.5"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@biomejs/biome": "^1.7.3",
|
"@biomejs/biome": "^1.7.3",
|
||||||
|
|||||||
Generated
+3466
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,13 @@
|
|||||||
|
# Block resolution of versions less than 24h old (Shai-Hulud window).
|
||||||
|
# This is the pnpm 11 default but pinned here so it's visible to
|
||||||
|
# reviewers and survives a future pnpm major flipping the default.
|
||||||
|
minimumReleaseAge: 1440
|
||||||
|
|
||||||
|
# Fail install if a transitive dep tries to run an unapproved script.
|
||||||
|
strictDepBuilds: true
|
||||||
|
|
||||||
|
allowBuilds:
|
||||||
|
'@biomejs/biome': true
|
||||||
|
onnxruntime-node: true
|
||||||
|
protobufjs: true
|
||||||
|
sharp: true
|
||||||
@@ -16,6 +16,18 @@ import {
|
|||||||
} from "./arrow";
|
} from "./arrow";
|
||||||
import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
|
import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
|
||||||
import { Connection as LanceDbConnection } from "./native";
|
import { Connection as LanceDbConnection } from "./native";
|
||||||
|
import type {
|
||||||
|
CreateNamespaceResponse,
|
||||||
|
DescribeNamespaceResponse,
|
||||||
|
DropNamespaceResponse,
|
||||||
|
ListNamespacesResponse,
|
||||||
|
} from "./native";
|
||||||
|
export type {
|
||||||
|
CreateNamespaceResponse,
|
||||||
|
DescribeNamespaceResponse,
|
||||||
|
DropNamespaceResponse,
|
||||||
|
ListNamespacesResponse,
|
||||||
|
};
|
||||||
import { sanitizeTable } from "./sanitize";
|
import { sanitizeTable } from "./sanitize";
|
||||||
import { LocalTable, Table } from "./table";
|
import { LocalTable, Table } from "./table";
|
||||||
|
|
||||||
@@ -110,6 +122,28 @@ export interface TableNamesOptions {
|
|||||||
/** An optional limit to the number of results to return. */
|
/** An optional limit to the number of results to return. */
|
||||||
limit?: number;
|
limit?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface ListNamespacesOptions {
|
||||||
|
/** Token from a previous response for pagination. */
|
||||||
|
pageToken?: string;
|
||||||
|
/** An optional limit to the number of results to return. */
|
||||||
|
limit?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CreateNamespaceOptions {
|
||||||
|
/** Creation mode. */
|
||||||
|
mode?: "create" | "exist_ok" | "overwrite";
|
||||||
|
/** Properties to set on the new namespace. */
|
||||||
|
properties?: Record<string, string>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DropNamespaceOptions {
|
||||||
|
/** Whether to skip if the namespace doesn't exist, or fail. */
|
||||||
|
mode?: "skip" | "fail";
|
||||||
|
/** Refuse to drop if non-empty (restrict) or drop recursively (cascade). */
|
||||||
|
behavior?: "restrict" | "cascade";
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A LanceDB Connection that allows you to open tables and create new ones.
|
* A LanceDB Connection that allows you to open tables and create new ones.
|
||||||
*
|
*
|
||||||
@@ -268,6 +302,69 @@ export abstract class Connection {
|
|||||||
*/
|
*/
|
||||||
abstract dropAllTables(namespacePath?: string[]): Promise<void>;
|
abstract dropAllTables(namespacePath?: string[]): Promise<void>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Describe a namespace, returning its properties.
|
||||||
|
*
|
||||||
|
* @param {string[]} namespacePath - The namespace path to describe, in
|
||||||
|
* parent → child order, e.g. `["analytics", "sales"]`.
|
||||||
|
* @returns {Promise<DescribeNamespaceResponse>} The namespace's properties
|
||||||
|
* (may be undefined if the namespace has none).
|
||||||
|
*/
|
||||||
|
abstract describeNamespace(
|
||||||
|
namespacePath: string[],
|
||||||
|
): Promise<DescribeNamespaceResponse>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* List the immediate child namespaces under the given parent.
|
||||||
|
*
|
||||||
|
* Results may be paginated. To retrieve subsequent pages, pass the
|
||||||
|
* `pageToken` returned by a previous call.
|
||||||
|
*
|
||||||
|
* @param {string[]} namespacePath - The parent namespace path. Defaults
|
||||||
|
* to the root namespace if omitted.
|
||||||
|
* @param {Partial<ListNamespacesOptions>} options - Pagination options
|
||||||
|
* (`pageToken`, `limit`).
|
||||||
|
* @returns {Promise<ListNamespacesResponse>} Child namespace names and
|
||||||
|
* an optional token for fetching the next page.
|
||||||
|
*/
|
||||||
|
abstract listNamespaces(
|
||||||
|
namespacePath?: string[],
|
||||||
|
options?: Partial<ListNamespacesOptions>,
|
||||||
|
): Promise<ListNamespacesResponse>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new namespace at the given path.
|
||||||
|
*
|
||||||
|
* @param {string[]} namespacePath - The namespace path to create.
|
||||||
|
* @param {Partial<CreateNamespaceOptions>} options - Creation `mode`
|
||||||
|
* ("create" | "exist_ok" | "overwrite") and optional `properties`
|
||||||
|
* to attach to the namespace.
|
||||||
|
* @returns {Promise<CreateNamespaceResponse>} The properties of the
|
||||||
|
* created namespace and an optional transaction id.
|
||||||
|
*/
|
||||||
|
abstract createNamespace(
|
||||||
|
namespacePath: string[],
|
||||||
|
options?: Partial<CreateNamespaceOptions>,
|
||||||
|
): Promise<CreateNamespaceResponse>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Drop a namespace.
|
||||||
|
*
|
||||||
|
* Use `behavior: "cascade"` to also drop everything contained in the
|
||||||
|
* namespace (sub-namespaces and tables). The default `"restrict"`
|
||||||
|
* behavior refuses to drop a non-empty namespace.
|
||||||
|
*
|
||||||
|
* @param {string[]} namespacePath - The namespace path to drop.
|
||||||
|
* @param {Partial<DropNamespaceOptions>} options - `mode` ("skip" | "fail"
|
||||||
|
* for missing-namespace handling) and `behavior` ("restrict" | "cascade").
|
||||||
|
* @returns {Promise<DropNamespaceResponse>} Any properties returned by
|
||||||
|
* the server and an optional transaction id.
|
||||||
|
*/
|
||||||
|
abstract dropNamespace(
|
||||||
|
namespacePath: string[],
|
||||||
|
options?: Partial<DropNamespaceOptions>,
|
||||||
|
): Promise<DropNamespaceResponse>;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Clone a table from a source table.
|
* Clone a table from a source table.
|
||||||
*
|
*
|
||||||
@@ -515,6 +612,45 @@ export class LocalConnection extends Connection {
|
|||||||
async dropAllTables(namespacePath?: string[]): Promise<void> {
|
async dropAllTables(namespacePath?: string[]): Promise<void> {
|
||||||
return this.inner.dropAllTables(namespacePath ?? []);
|
return this.inner.dropAllTables(namespacePath ?? []);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
describeNamespace(
|
||||||
|
namespacePath: string[],
|
||||||
|
): Promise<DescribeNamespaceResponse> {
|
||||||
|
return this.inner.describeNamespace(namespacePath);
|
||||||
|
}
|
||||||
|
|
||||||
|
listNamespaces(
|
||||||
|
namespacePath?: string[],
|
||||||
|
options?: Partial<ListNamespacesOptions>,
|
||||||
|
): Promise<ListNamespacesResponse> {
|
||||||
|
return this.inner.listNamespaces(
|
||||||
|
namespacePath ?? [],
|
||||||
|
options?.pageToken,
|
||||||
|
options?.limit,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
createNamespace(
|
||||||
|
namespacePath: string[],
|
||||||
|
options?: Partial<CreateNamespaceOptions>,
|
||||||
|
): Promise<CreateNamespaceResponse> {
|
||||||
|
return this.inner.createNamespace(
|
||||||
|
namespacePath,
|
||||||
|
options?.mode,
|
||||||
|
options?.properties,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
dropNamespace(
|
||||||
|
namespacePath: string[],
|
||||||
|
options?: Partial<DropNamespaceOptions>,
|
||||||
|
): Promise<DropNamespaceResponse> {
|
||||||
|
return this.inner.dropNamespace(
|
||||||
|
namespacePath,
|
||||||
|
options?.mode,
|
||||||
|
options?.behavior,
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import {
|
|||||||
} from "./connection";
|
} from "./connection";
|
||||||
|
|
||||||
import {
|
import {
|
||||||
|
ConnectNamespaceOptions,
|
||||||
ConnectionOptions,
|
ConnectionOptions,
|
||||||
Connection as LanceDbConnection,
|
Connection as LanceDbConnection,
|
||||||
JsHeaderProvider as NativeJsHeaderProvider,
|
JsHeaderProvider as NativeJsHeaderProvider,
|
||||||
@@ -22,6 +23,7 @@ export { JsHeaderProvider as NativeJsHeaderProvider } from "./native.js";
|
|||||||
export {
|
export {
|
||||||
AddColumnsSql,
|
AddColumnsSql,
|
||||||
ConnectionOptions,
|
ConnectionOptions,
|
||||||
|
ConnectNamespaceOptions,
|
||||||
IndexStatistics,
|
IndexStatistics,
|
||||||
IndexConfig,
|
IndexConfig,
|
||||||
ClientConfig,
|
ClientConfig,
|
||||||
@@ -62,6 +64,13 @@ export {
|
|||||||
CreateTableOptions,
|
CreateTableOptions,
|
||||||
TableNamesOptions,
|
TableNamesOptions,
|
||||||
OpenTableOptions,
|
OpenTableOptions,
|
||||||
|
ListNamespacesOptions,
|
||||||
|
CreateNamespaceOptions,
|
||||||
|
DropNamespaceOptions,
|
||||||
|
ListNamespacesResponse,
|
||||||
|
CreateNamespaceResponse,
|
||||||
|
DropNamespaceResponse,
|
||||||
|
DescribeNamespaceResponse,
|
||||||
} from "./connection";
|
} from "./connection";
|
||||||
|
|
||||||
export { Session } from "./native.js";
|
export { Session } from "./native.js";
|
||||||
@@ -293,3 +302,197 @@ export async function connect(
|
|||||||
);
|
);
|
||||||
return new LocalConnection(nativeConn);
|
return new LocalConnection(nativeConn);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Configuration for the built-in directory namespace (`"dir"`).
|
||||||
|
*
|
||||||
|
* The directory namespace stores tables under a single root path (local
|
||||||
|
* filesystem or object storage URI). See
|
||||||
|
* {@link https://docs.lancedb.com/namespaces} for the documented surface;
|
||||||
|
* less-common knobs live under {@link DirNamespaceConfig.extraProperties}.
|
||||||
|
*/
|
||||||
|
export interface DirNamespaceConfig {
|
||||||
|
/** Root path or URI containing the LanceDB tables. */
|
||||||
|
root: string;
|
||||||
|
/**
|
||||||
|
* Whether to maintain a namespace manifest at the root. Required for
|
||||||
|
* child namespaces. Defaults to true on the impl side.
|
||||||
|
*/
|
||||||
|
manifestEnabled?: boolean;
|
||||||
|
/**
|
||||||
|
* Additional raw properties passed verbatim to the namespace
|
||||||
|
* implementation (e.g. `storage.*`, `credential_vendor.*`). Typed
|
||||||
|
* fields above take precedence on key collision.
|
||||||
|
*/
|
||||||
|
extraProperties?: Record<string, string>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Configuration for the built-in REST namespace (`"rest"`).
|
||||||
|
*
|
||||||
|
* The REST namespace talks to a remote catalog server over HTTP. See
|
||||||
|
* {@link https://docs.lancedb.com/namespaces} for the documented surface;
|
||||||
|
* less-common knobs (TLS, metrics) live under
|
||||||
|
* {@link RestNamespaceConfig.extraProperties}.
|
||||||
|
*/
|
||||||
|
export interface RestNamespaceConfig {
|
||||||
|
/** Catalog endpoint URL. */
|
||||||
|
uri: string;
|
||||||
|
/**
|
||||||
|
* HTTP headers forwarded with each request. Keys are passed through
|
||||||
|
* as-is (e.g. `"x-api-key"`, `"Authorization"`).
|
||||||
|
*/
|
||||||
|
headers?: Record<string, string>;
|
||||||
|
/**
|
||||||
|
* Additional raw properties passed verbatim to the namespace
|
||||||
|
* implementation (e.g. `tls.*`, `ops_metrics_enabled`, `delimiter`).
|
||||||
|
* Typed fields above take precedence on key collision.
|
||||||
|
*/
|
||||||
|
extraProperties?: Record<string, string>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function dirConfigToProperties(
|
||||||
|
config: DirNamespaceConfig,
|
||||||
|
): Record<string, string> {
|
||||||
|
// Spread the whole input so that unknown keys (e.g. a raw `manifest_enabled`
|
||||||
|
// passed via the dynamic-impl path) flow through instead of being dropped.
|
||||||
|
// Typed transformations layer on top.
|
||||||
|
const { manifestEnabled, extraProperties, ...rest } = config;
|
||||||
|
const properties: Record<string, string> = {
|
||||||
|
...(extraProperties ?? {}),
|
||||||
|
...(rest as Record<string, string>),
|
||||||
|
};
|
||||||
|
if (manifestEnabled !== undefined) {
|
||||||
|
properties.manifest_enabled = String(manifestEnabled);
|
||||||
|
}
|
||||||
|
return properties;
|
||||||
|
}
|
||||||
|
|
||||||
|
function restConfigToProperties(
|
||||||
|
config: RestNamespaceConfig,
|
||||||
|
): Record<string, string> {
|
||||||
|
const { headers, extraProperties, ...rest } = config;
|
||||||
|
const properties: Record<string, string> = {
|
||||||
|
...(extraProperties ?? {}),
|
||||||
|
...(rest as Record<string, string>),
|
||||||
|
};
|
||||||
|
if (headers) {
|
||||||
|
for (const [name, value] of Object.entries(headers)) {
|
||||||
|
properties[`headers.${name}`] = value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return properties;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Connect to a LanceDB database through a namespace.
|
||||||
|
*
|
||||||
|
* Unlike {@link connect}, which routes by URI scheme (local path vs.
|
||||||
|
* `db://` cloud), `connectNamespace` always returns a namespace-backed
|
||||||
|
* connection. The `implName` selects the namespace implementation:
|
||||||
|
*
|
||||||
|
* - `"dir"` — directory namespace, configured with {@link DirNamespaceConfig}.
|
||||||
|
* - `"rest"` — remote REST catalog, configured with {@link RestNamespaceConfig}.
|
||||||
|
* - Any other string — full module path for a custom implementation,
|
||||||
|
* configured with a free-form string-keyed `properties` map.
|
||||||
|
*
|
||||||
|
* @example Typed dir namespace
|
||||||
|
* ```ts
|
||||||
|
* const db = await connectNamespace("dir", { root: "/path/to/db" });
|
||||||
|
* await db.createTable("users", [{ id: 1 }]);
|
||||||
|
* ```
|
||||||
|
*
|
||||||
|
* @example Typed REST namespace with auth headers
|
||||||
|
* ```ts
|
||||||
|
* const db = await connectNamespace("rest", {
|
||||||
|
* uri: "https://catalog.example.com",
|
||||||
|
* headers: { "x-api-key": process.env.CATALOG_KEY ?? "" },
|
||||||
|
* });
|
||||||
|
* ```
|
||||||
|
*
|
||||||
|
* @example Custom implementation with raw properties
|
||||||
|
* ```ts
|
||||||
|
* const db = await connectNamespace("my.custom.Namespace", {
|
||||||
|
* endpoint: "...",
|
||||||
|
* });
|
||||||
|
* ```
|
||||||
|
*/
|
||||||
|
export function connectNamespace(
|
||||||
|
implName: "dir",
|
||||||
|
config: DirNamespaceConfig,
|
||||||
|
options?: Partial<ConnectNamespaceOptions>,
|
||||||
|
): Promise<Connection>;
|
||||||
|
/**
|
||||||
|
* Connect through the built-in REST namespace.
|
||||||
|
*
|
||||||
|
* Configured with {@link RestNamespaceConfig}. See the function-level
|
||||||
|
* documentation above for the full surface, examples, and how this
|
||||||
|
* relates to {@link connect}.
|
||||||
|
*
|
||||||
|
* @example
|
||||||
|
* ```ts
|
||||||
|
* const db = await connectNamespace("rest", {
|
||||||
|
* uri: "https://catalog.example.com",
|
||||||
|
* headers: { "x-api-key": process.env.CATALOG_KEY ?? "" },
|
||||||
|
* });
|
||||||
|
* ```
|
||||||
|
*/
|
||||||
|
export function connectNamespace(
|
||||||
|
implName: "rest",
|
||||||
|
config: RestNamespaceConfig,
|
||||||
|
options?: Partial<ConnectNamespaceOptions>,
|
||||||
|
): Promise<Connection>;
|
||||||
|
/**
|
||||||
|
* Connect through a custom namespace implementation by full module path,
|
||||||
|
* configured with a free-form string-keyed `properties` map. Use the
|
||||||
|
* typed overloads above for the built-in `"dir"` and `"rest"` impls.
|
||||||
|
*
|
||||||
|
* See the function-level documentation above for examples and how this
|
||||||
|
* relates to {@link connect}.
|
||||||
|
*
|
||||||
|
* @example
|
||||||
|
* ```ts
|
||||||
|
* const db = await connectNamespace("my.custom.Namespace", {
|
||||||
|
* endpoint: "...",
|
||||||
|
* });
|
||||||
|
* ```
|
||||||
|
*/
|
||||||
|
export function connectNamespace(
|
||||||
|
implName: string,
|
||||||
|
properties: Record<string, string>,
|
||||||
|
options?: Partial<ConnectNamespaceOptions>,
|
||||||
|
): Promise<Connection>;
|
||||||
|
export async function connectNamespace(
|
||||||
|
implName: string,
|
||||||
|
configOrProperties:
|
||||||
|
| DirNamespaceConfig
|
||||||
|
| RestNamespaceConfig
|
||||||
|
| Record<string, string>,
|
||||||
|
options?: Partial<ConnectNamespaceOptions>,
|
||||||
|
): Promise<Connection> {
|
||||||
|
let properties: Record<string, string>;
|
||||||
|
if (implName === "dir") {
|
||||||
|
properties = dirConfigToProperties(
|
||||||
|
configOrProperties as DirNamespaceConfig,
|
||||||
|
);
|
||||||
|
} else if (implName === "rest") {
|
||||||
|
properties = restConfigToProperties(
|
||||||
|
configOrProperties as RestNamespaceConfig,
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
properties = configOrProperties as Record<string, string>;
|
||||||
|
}
|
||||||
|
|
||||||
|
const finalOptions: ConnectNamespaceOptions = (options ??
|
||||||
|
{}) as ConnectNamespaceOptions;
|
||||||
|
finalOptions.storageOptions = cleanseStorageOptions(
|
||||||
|
finalOptions.storageOptions,
|
||||||
|
);
|
||||||
|
|
||||||
|
const nativeConn = await LanceDbConnection.newWithNamespace(
|
||||||
|
implName,
|
||||||
|
properties,
|
||||||
|
finalOptions,
|
||||||
|
);
|
||||||
|
return new LocalConnection(nativeConn);
|
||||||
|
}
|
||||||
|
|||||||
@@ -87,6 +87,23 @@ export class MergeInsertBuilder {
|
|||||||
this.#schema,
|
this.#schema,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Skip the automatic cleanup of old dataset versions that would otherwise
|
||||||
|
* run as part of this merge insert's commit. Forwards to
|
||||||
|
* `MergeInsertBuilder::skip_auto_cleanup` in lance-core.
|
||||||
|
*
|
||||||
|
* Useful for high-frequency writers that prefer to manage version cleanup
|
||||||
|
* themselves, or writers without delete permissions on the underlying storage.
|
||||||
|
*
|
||||||
|
* @param skip - If true, the auto-cleanup step is skipped at commit time.
|
||||||
|
*/
|
||||||
|
skipAutoCleanup(skip: boolean): MergeInsertBuilder {
|
||||||
|
return new MergeInsertBuilder(
|
||||||
|
this.#native.skipAutoCleanup(skip),
|
||||||
|
this.#schema,
|
||||||
|
);
|
||||||
|
}
|
||||||
/**
|
/**
|
||||||
* Executes the merge insert operation
|
* Executes the merge insert operation
|
||||||
*
|
*
|
||||||
|
|||||||
+13
-1
@@ -56,6 +56,18 @@ export interface AddDataOptions {
|
|||||||
* If "overwrite" then the new data will replace the existing data in the table.
|
* If "overwrite" then the new data will replace the existing data in the table.
|
||||||
*/
|
*/
|
||||||
mode: "append" | "overwrite";
|
mode: "append" | "overwrite";
|
||||||
|
/**
|
||||||
|
* If true, skip the automatic cleanup of old dataset versions that would
|
||||||
|
* otherwise run as part of this write's commit. Forwards to
|
||||||
|
* `WriteParams.skip_auto_cleanup` in lance-core.
|
||||||
|
*
|
||||||
|
* Useful for high-frequency writers that prefer to manage version cleanup
|
||||||
|
* themselves (for example, via a separate periodic optimize job), or for
|
||||||
|
* writers that don't have delete permissions on the underlying storage.
|
||||||
|
*
|
||||||
|
* Defaults to false.
|
||||||
|
*/
|
||||||
|
skipAutoCleanup?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface UpdateOptions {
|
export interface UpdateOptions {
|
||||||
@@ -636,7 +648,7 @@ export class LocalTable extends Table {
|
|||||||
const schema = await this.schema();
|
const schema = await this.schema();
|
||||||
|
|
||||||
const buffer = await fromDataToBuffer(data, undefined, schema);
|
const buffer = await fromDataToBuffer(data, undefined, schema);
|
||||||
return await this.inner.add(buffer, mode);
|
return await this.inner.add(buffer, mode, options?.skipAutoCleanup);
|
||||||
}
|
}
|
||||||
|
|
||||||
async update(
|
async update(
|
||||||
|
|||||||
Generated
-10452
File diff suppressed because it is too large
Load Diff
+15
-14
@@ -38,15 +38,15 @@
|
|||||||
"url": "https://github.com/lancedb/lancedb"
|
"url": "https://github.com/lancedb/lancedb"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@aws-sdk/client-dynamodb": "^3.33.0",
|
"@aws-sdk/client-dynamodb": "3.1003.0",
|
||||||
"@aws-sdk/client-kms": "^3.33.0",
|
"@aws-sdk/client-kms": "3.1003.0",
|
||||||
"@aws-sdk/client-s3": "^3.33.0",
|
"@aws-sdk/client-s3": "3.1003.0",
|
||||||
"@biomejs/biome": "^1.7.3",
|
"@biomejs/biome": "^1.7.3",
|
||||||
"@jest/globals": "^29.7.0",
|
"@jest/globals": "^29.7.0",
|
||||||
"@napi-rs/cli": "^3.5.1",
|
"@napi-rs/cli": "3.5.1",
|
||||||
"@types/axios": "^0.14.0",
|
"@types/axios": "^0.14.0",
|
||||||
"@types/jest": "^29.1.2",
|
"@types/jest": "^29.1.2",
|
||||||
"@types/node": "^22.7.4",
|
"@types/node": "22.7.4",
|
||||||
"@types/tmp": "^0.2.6",
|
"@types/tmp": "^0.2.6",
|
||||||
"apache-arrow-15": "npm:apache-arrow@15.0.0",
|
"apache-arrow-15": "npm:apache-arrow@15.0.0",
|
||||||
"apache-arrow-16": "npm:apache-arrow@16.0.0",
|
"apache-arrow-16": "npm:apache-arrow@16.0.0",
|
||||||
@@ -57,9 +57,9 @@
|
|||||||
"shx": "^0.3.4",
|
"shx": "^0.3.4",
|
||||||
"tmp": "^0.2.3",
|
"tmp": "^0.2.3",
|
||||||
"ts-jest": "^29.1.2",
|
"ts-jest": "^29.1.2",
|
||||||
"typedoc": "^0.26.4",
|
"typedoc": "0.26.4",
|
||||||
"typedoc-plugin-markdown": "^4.2.1",
|
"typedoc-plugin-markdown": "4.2.1",
|
||||||
"typescript": "^5.5.4",
|
"typescript": "5.5.4",
|
||||||
"typescript-eslint": "^7.1.0"
|
"typescript-eslint": "^7.1.0"
|
||||||
},
|
},
|
||||||
"ava": {
|
"ava": {
|
||||||
@@ -68,15 +68,16 @@
|
|||||||
"engines": {
|
"engines": {
|
||||||
"node": ">= 18"
|
"node": ">= 18"
|
||||||
},
|
},
|
||||||
|
"packageManager": "pnpm@11.1.1",
|
||||||
"cpu": ["x64", "arm64"],
|
"cpu": ["x64", "arm64"],
|
||||||
"os": ["darwin", "linux", "win32"],
|
"os": ["darwin", "linux", "win32"],
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"artifacts": "napi artifacts",
|
"artifacts": "napi artifacts",
|
||||||
"build:debug": "napi build --platform --dts ../lancedb/native.d.ts --js ../lancedb/native.js --output-dir lancedb",
|
"build:debug": "napi build --platform --dts ../lancedb/native.d.ts --js ../lancedb/native.js --output-dir lancedb",
|
||||||
"postbuild:debug": "shx mkdir -p dist && shx cp lancedb/*.node dist/",
|
"postbuild:debug": "shx mkdir -p dist && shx cp lancedb/*.node dist/ && node -e \"require('fs').writeFileSync('dist/package.json', JSON.stringify({name:'@lancedb/lancedb',type:'commonjs'}))\"",
|
||||||
"build:release": "napi build --platform --release --dts ../lancedb/native.d.ts --js ../lancedb/native.js --output-dir dist",
|
"build:release": "napi build --platform --release --dts ../lancedb/native.d.ts --js ../lancedb/native.js --output-dir dist",
|
||||||
"build": "npm run build:debug && npm run tsc",
|
"build": "pnpm build:debug && pnpm tsc",
|
||||||
"build-release": "npm run build:release && npm run tsc",
|
"build-release": "pnpm build:release && pnpm tsc",
|
||||||
"tsc": "tsc -b",
|
"tsc": "tsc -b",
|
||||||
"posttsc": "shx cp lancedb/native.d.ts dist/native.d.ts",
|
"posttsc": "shx cp lancedb/native.d.ts dist/native.d.ts",
|
||||||
"lint-ci": "biome ci .",
|
"lint-ci": "biome ci .",
|
||||||
@@ -86,7 +87,7 @@
|
|||||||
"lint-fix": "biome check --write . && biome format --write .",
|
"lint-fix": "biome check --write . && biome format --write .",
|
||||||
"prepublishOnly": "napi prepublish -t npm",
|
"prepublishOnly": "napi prepublish -t npm",
|
||||||
"test": "jest --verbose",
|
"test": "jest --verbose",
|
||||||
"integration": "S3_TEST=1 npm run test",
|
"integration": "S3_TEST=1 pnpm test",
|
||||||
"universal": "napi universalize",
|
"universal": "napi universalize",
|
||||||
"version": "napi version"
|
"version": "napi version"
|
||||||
},
|
},
|
||||||
@@ -94,8 +95,8 @@
|
|||||||
"reflect-metadata": "^0.2.2"
|
"reflect-metadata": "^0.2.2"
|
||||||
},
|
},
|
||||||
"optionalDependencies": {
|
"optionalDependencies": {
|
||||||
"@huggingface/transformers": "^3.0.2",
|
"@huggingface/transformers": "3.0.2",
|
||||||
"openai": "^4.29.2"
|
"openai": "4.29.2"
|
||||||
},
|
},
|
||||||
"peerDependencies": {
|
"peerDependencies": {
|
||||||
"apache-arrow": ">=15.0.0 <=18.1.0"
|
"apache-arrow": ">=15.0.0 <=18.1.0"
|
||||||
|
|||||||
Generated
+7317
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,18 @@
|
|||||||
|
# Flat node_modules layout. The @napi-rs/cli build step fails to locate
|
||||||
|
# the cdylib artifact under pnpm's isolated layout; the hoisted linker
|
||||||
|
# mirrors npm's structure and unblocks the native build.
|
||||||
|
nodeLinker: hoisted
|
||||||
|
|
||||||
|
# Block resolution of versions less than 24h old (Shai-Hulud window).
|
||||||
|
# This is the pnpm 11 default but pinned here so it's visible to
|
||||||
|
# reviewers and survives a future pnpm major flipping the default.
|
||||||
|
minimumReleaseAge: 1440
|
||||||
|
|
||||||
|
# Fail install if a transitive dep tries to run an unapproved script.
|
||||||
|
strictDepBuilds: true
|
||||||
|
|
||||||
|
allowBuilds:
|
||||||
|
'@biomejs/biome': true
|
||||||
|
onnxruntime-node: true
|
||||||
|
protobufjs: true
|
||||||
|
sharp: true
|
||||||
+187
-1
@@ -8,12 +8,16 @@ use lancedb::database::{CreateTableMode, Database};
|
|||||||
use napi::bindgen_prelude::*;
|
use napi::bindgen_prelude::*;
|
||||||
use napi_derive::*;
|
use napi_derive::*;
|
||||||
|
|
||||||
|
use crate::ConnectNamespaceOptions;
|
||||||
use crate::ConnectionOptions;
|
use crate::ConnectionOptions;
|
||||||
use crate::error::NapiErrorExt;
|
use crate::error::NapiErrorExt;
|
||||||
use crate::header::JsHeaderProvider;
|
use crate::header::JsHeaderProvider;
|
||||||
use crate::table::Table;
|
use crate::table::Table;
|
||||||
use lancedb::connection::{ConnectBuilder, Connection as LanceDBConnection};
|
use lancedb::connection::{ConnectBuilder, Connection as LanceDBConnection, connect_namespace};
|
||||||
|
|
||||||
|
use lance_namespace::models::{
|
||||||
|
CreateNamespaceRequest, DescribeNamespaceRequest, DropNamespaceRequest, ListNamespacesRequest,
|
||||||
|
};
|
||||||
use lancedb::ipc::{ipc_file_to_batches, ipc_file_to_schema};
|
use lancedb::ipc::{ipc_file_to_batches, ipc_file_to_schema};
|
||||||
|
|
||||||
#[napi]
|
#[napi]
|
||||||
@@ -21,6 +25,29 @@ pub struct Connection {
|
|||||||
inner: Option<LanceDBConnection>,
|
inner: Option<LanceDBConnection>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[napi(object)]
|
||||||
|
pub struct DescribeNamespaceResponse {
|
||||||
|
pub properties: Option<HashMap<String, String>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[napi(object)]
|
||||||
|
pub struct ListNamespacesResponse {
|
||||||
|
pub namespaces: Vec<String>,
|
||||||
|
pub page_token: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[napi(object)]
|
||||||
|
pub struct CreateNamespaceResponse {
|
||||||
|
pub properties: Option<HashMap<String, String>>,
|
||||||
|
pub transaction_id: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[napi(object)]
|
||||||
|
pub struct DropNamespaceResponse {
|
||||||
|
pub properties: Option<HashMap<String, String>>,
|
||||||
|
pub transaction_id: Option<Vec<String>>,
|
||||||
|
}
|
||||||
|
|
||||||
impl Connection {
|
impl Connection {
|
||||||
pub(crate) fn inner_new(inner: LanceDBConnection) -> Self {
|
pub(crate) fn inner_new(inner: LanceDBConnection) -> Self {
|
||||||
Self { inner: Some(inner) }
|
Self { inner: Some(inner) }
|
||||||
@@ -106,6 +133,39 @@ impl Connection {
|
|||||||
Ok(Self::inner_new(builder.execute().await.default_error()?))
|
Ok(Self::inner_new(builder.execute().await.default_error()?))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Create a new Connection instance backed by a namespace implementation.
|
||||||
|
#[napi(factory)]
|
||||||
|
pub async fn new_with_namespace(
|
||||||
|
impl_name: String,
|
||||||
|
properties: HashMap<String, String>,
|
||||||
|
options: ConnectNamespaceOptions,
|
||||||
|
) -> napi::Result<Self> {
|
||||||
|
if impl_name.is_empty() {
|
||||||
|
return Err(napi::Error::from_reason(
|
||||||
|
"implName must be a non-empty string",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut builder = connect_namespace(&impl_name, properties);
|
||||||
|
if let Some(interval) = options.read_consistency_interval {
|
||||||
|
builder =
|
||||||
|
builder.read_consistency_interval(std::time::Duration::from_secs_f64(interval));
|
||||||
|
}
|
||||||
|
if let Some(storage_options) = options.storage_options {
|
||||||
|
for (key, value) in storage_options {
|
||||||
|
builder = builder.storage_option(key, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(namespace_client_properties) = options.namespace_client_properties {
|
||||||
|
builder = builder.namespace_client_properties(namespace_client_properties);
|
||||||
|
}
|
||||||
|
if let Some(session) = options.session {
|
||||||
|
builder = builder.session(session.inner.clone());
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(Self::inner_new(builder.execute().await.default_error()?))
|
||||||
|
}
|
||||||
|
|
||||||
#[napi]
|
#[napi]
|
||||||
pub fn display(&self) -> napi::Result<String> {
|
pub fn display(&self) -> napi::Result<String> {
|
||||||
Ok(self.get_inner()?.to_string())
|
Ok(self.get_inner()?.to_string())
|
||||||
@@ -273,4 +333,130 @@ impl Connection {
|
|||||||
let ns = namespace_path.unwrap_or_default();
|
let ns = namespace_path.unwrap_or_default();
|
||||||
self.get_inner()?.drop_all_tables(&ns).await.default_error()
|
self.get_inner()?.drop_all_tables(&ns).await.default_error()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[napi(catch_unwind)]
|
||||||
|
/// Describe a namespace and return its properties.
|
||||||
|
pub async fn describe_namespace(
|
||||||
|
&self,
|
||||||
|
namespace_path: Vec<String>,
|
||||||
|
) -> napi::Result<DescribeNamespaceResponse> {
|
||||||
|
let req = DescribeNamespaceRequest {
|
||||||
|
id: Some(namespace_path),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let resp = self
|
||||||
|
.get_inner()?
|
||||||
|
.describe_namespace(req)
|
||||||
|
.await
|
||||||
|
.default_error()?;
|
||||||
|
Ok(DescribeNamespaceResponse {
|
||||||
|
properties: resp.properties,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[napi(catch_unwind)]
|
||||||
|
/// List child namespaces under the given namespace path
|
||||||
|
pub async fn list_namespaces(
|
||||||
|
&self,
|
||||||
|
namespace_path: Option<Vec<String>>,
|
||||||
|
page_token: Option<String>,
|
||||||
|
limit: Option<u32>,
|
||||||
|
) -> napi::Result<ListNamespacesResponse> {
|
||||||
|
let req = ListNamespacesRequest {
|
||||||
|
id: namespace_path,
|
||||||
|
page_token,
|
||||||
|
limit: limit.map(|l| l as i32),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let resp = self
|
||||||
|
.get_inner()?
|
||||||
|
.list_namespaces(req)
|
||||||
|
.await
|
||||||
|
.default_error()?;
|
||||||
|
Ok(ListNamespacesResponse {
|
||||||
|
namespaces: resp.namespaces,
|
||||||
|
page_token: resp.page_token,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[napi(catch_unwind)]
|
||||||
|
/// Create a new namespace with optional properties.
|
||||||
|
pub async fn create_namespace(
|
||||||
|
&self,
|
||||||
|
namespace_path: Vec<String>,
|
||||||
|
mode: Option<String>,
|
||||||
|
properties: Option<HashMap<String, String>>,
|
||||||
|
) -> napi::Result<CreateNamespaceResponse> {
|
||||||
|
let mode_str = mode
|
||||||
|
.map(|m| match m.to_lowercase().as_str() {
|
||||||
|
"create" => Ok("Create".to_string()),
|
||||||
|
"exist_ok" => Ok("ExistOk".to_string()),
|
||||||
|
"overwrite" => Ok("Overwrite".to_string()),
|
||||||
|
_ => Err(napi::Error::from_reason(format!(
|
||||||
|
"Invalid mode '{}': expected one of 'create', 'exist_ok', 'overwrite'",
|
||||||
|
m
|
||||||
|
))),
|
||||||
|
})
|
||||||
|
.transpose()?;
|
||||||
|
let req = CreateNamespaceRequest {
|
||||||
|
id: Some(namespace_path),
|
||||||
|
mode: mode_str,
|
||||||
|
properties,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let resp = self
|
||||||
|
.get_inner()?
|
||||||
|
.create_namespace(req)
|
||||||
|
.await
|
||||||
|
.default_error()?;
|
||||||
|
Ok(CreateNamespaceResponse {
|
||||||
|
properties: resp.properties,
|
||||||
|
transaction_id: resp.transaction_id,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[napi(catch_unwind)]
|
||||||
|
/// Drop a namespace.
|
||||||
|
pub async fn drop_namespace(
|
||||||
|
&self,
|
||||||
|
namespace_path: Vec<String>,
|
||||||
|
mode: Option<String>,
|
||||||
|
behavior: Option<String>,
|
||||||
|
) -> napi::Result<DropNamespaceResponse> {
|
||||||
|
let mode_str = mode
|
||||||
|
.map(|m| match m.to_lowercase().as_str() {
|
||||||
|
"skip" => Ok("Skip".to_string()),
|
||||||
|
"fail" => Ok("Fail".to_string()),
|
||||||
|
_ => Err(napi::Error::from_reason(format!(
|
||||||
|
"Invalid mode '{}': expected one of 'skip', 'fail'",
|
||||||
|
m
|
||||||
|
))),
|
||||||
|
})
|
||||||
|
.transpose()?;
|
||||||
|
let behavior_str = behavior
|
||||||
|
.map(|b| match b.to_lowercase().as_str() {
|
||||||
|
"restrict" => Ok("Restrict".to_string()),
|
||||||
|
"cascade" => Ok("Cascade".to_string()),
|
||||||
|
_ => Err(napi::Error::from_reason(format!(
|
||||||
|
"Invalid behavior '{}': expected one of 'restrict', 'cascade'",
|
||||||
|
b
|
||||||
|
))),
|
||||||
|
})
|
||||||
|
.transpose()?;
|
||||||
|
let req = DropNamespaceRequest {
|
||||||
|
id: Some(namespace_path),
|
||||||
|
mode: mode_str,
|
||||||
|
behavior: behavior_str,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let resp = self
|
||||||
|
.get_inner()?
|
||||||
|
.drop_namespace(req)
|
||||||
|
.await
|
||||||
|
.default_error()?;
|
||||||
|
Ok(DropNamespaceResponse {
|
||||||
|
properties: resp.properties,
|
||||||
|
transaction_id: resp.transaction_id,
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -67,6 +67,26 @@ pub struct OpenTableOptions {
|
|||||||
pub storage_options: Option<HashMap<String, String>>,
|
pub storage_options: Option<HashMap<String, String>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[napi(object)]
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct ConnectNamespaceOptions {
|
||||||
|
/// The interval, in seconds, at which to check for updates to the table
|
||||||
|
/// from other processes. If None, then consistency is not checked. For
|
||||||
|
/// performance reasons, this is the default. For strong consistency, set
|
||||||
|
/// this to zero seconds. Then every read will check for updates from other
|
||||||
|
/// processes. As a compromise, you can set this to a non-zero value for
|
||||||
|
/// eventual consistency.
|
||||||
|
pub read_consistency_interval: Option<f64>,
|
||||||
|
/// Configuration for object storage. The available options are described
|
||||||
|
/// at https://docs.lancedb.com/storage/
|
||||||
|
pub storage_options: Option<HashMap<String, String>>,
|
||||||
|
/// Extra properties for the backing namespace client.
|
||||||
|
pub namespace_client_properties: Option<HashMap<String, String>>,
|
||||||
|
/// The session to use for this connection. Holds shared caches and other
|
||||||
|
/// session-specific state.
|
||||||
|
pub session: Option<session::Session>,
|
||||||
|
}
|
||||||
|
|
||||||
#[napi_derive::module_init]
|
#[napi_derive::module_init]
|
||||||
fn init() {
|
fn init() {
|
||||||
let env = Env::new()
|
let env = Env::new()
|
||||||
|
|||||||
@@ -50,6 +50,13 @@ impl NativeMergeInsertBuilder {
|
|||||||
this
|
this
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[napi]
|
||||||
|
pub fn skip_auto_cleanup(&self, skip: bool) -> Self {
|
||||||
|
let mut this = self.clone();
|
||||||
|
this.inner.skip_auto_cleanup(skip);
|
||||||
|
this
|
||||||
|
}
|
||||||
|
|
||||||
#[napi(catch_unwind)]
|
#[napi(catch_unwind)]
|
||||||
pub async fn execute(&self, buf: Buffer) -> napi::Result<MergeResult> {
|
pub async fn execute(&self, buf: Buffer) -> napi::Result<MergeResult> {
|
||||||
let data = ipc_file_to_batches(buf.to_vec())
|
let data = ipc_file_to_batches(buf.to_vec())
|
||||||
|
|||||||
+14
-2
@@ -6,7 +6,7 @@ use std::collections::HashMap;
|
|||||||
use lancedb::ipc::{ipc_file_to_batches, ipc_file_to_schema};
|
use lancedb::ipc::{ipc_file_to_batches, ipc_file_to_schema};
|
||||||
use lancedb::table::{
|
use lancedb::table::{
|
||||||
AddDataMode, ColumnAlteration as LanceColumnAlteration, Duration, NewColumnTransform,
|
AddDataMode, ColumnAlteration as LanceColumnAlteration, Duration, NewColumnTransform,
|
||||||
OptimizeAction, OptimizeOptions, Table as LanceDbTable,
|
OptimizeAction, OptimizeOptions, Table as LanceDbTable, WriteOptions,
|
||||||
};
|
};
|
||||||
use napi::bindgen_prelude::*;
|
use napi::bindgen_prelude::*;
|
||||||
use napi_derive::napi;
|
use napi_derive::napi;
|
||||||
@@ -68,7 +68,12 @@ impl Table {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[napi(catch_unwind)]
|
#[napi(catch_unwind)]
|
||||||
pub async fn add(&self, buf: Buffer, mode: String) -> napi::Result<AddResult> {
|
pub async fn add(
|
||||||
|
&self,
|
||||||
|
buf: Buffer,
|
||||||
|
mode: String,
|
||||||
|
skip_auto_cleanup: Option<bool>,
|
||||||
|
) -> napi::Result<AddResult> {
|
||||||
let batches = ipc_file_to_batches(buf.to_vec())
|
let batches = ipc_file_to_batches(buf.to_vec())
|
||||||
.map_err(|e| napi::Error::from_reason(format!("Failed to read IPC file: {}", e)))?;
|
.map_err(|e| napi::Error::from_reason(format!("Failed to read IPC file: {}", e)))?;
|
||||||
let batches = batches
|
let batches = batches
|
||||||
@@ -92,6 +97,13 @@ impl Table {
|
|||||||
return Err(napi::Error::from_reason(format!("Invalid mode: {}", mode)));
|
return Err(napi::Error::from_reason(format!("Invalid mode: {}", mode)));
|
||||||
};
|
};
|
||||||
|
|
||||||
|
if skip_auto_cleanup.unwrap_or(false) {
|
||||||
|
op = op.write_options(WriteOptions {
|
||||||
|
skip_auto_cleanup: true,
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
let res = op.execute().await.default_error()?;
|
let res = op.execute().await.default_error()?;
|
||||||
Ok(res.into())
|
Ok(res.into())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -849,6 +849,10 @@ impl ListingDatabase {
|
|||||||
write_params.mode = WriteMode::Overwrite;
|
write_params.mode = WriteMode::Overwrite;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if request.write_options.skip_auto_cleanup {
|
||||||
|
write_params.skip_auto_cleanup = true;
|
||||||
|
}
|
||||||
|
|
||||||
write_params.session = Some(self.session.clone());
|
write_params.session = Some(self.session.clone());
|
||||||
|
|
||||||
write_params
|
write_params
|
||||||
@@ -2034,6 +2038,7 @@ mod tests {
|
|||||||
}),
|
}),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
}),
|
}),
|
||||||
|
..Default::default()
|
||||||
};
|
};
|
||||||
|
|
||||||
let table = db
|
let table = db
|
||||||
@@ -2107,6 +2112,7 @@ mod tests {
|
|||||||
}),
|
}),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
}),
|
}),
|
||||||
|
..Default::default()
|
||||||
};
|
};
|
||||||
|
|
||||||
let table = db
|
let table = db
|
||||||
|
|||||||
@@ -414,6 +414,10 @@ impl Database for LanceNamespaceDatabase {
|
|||||||
params.mode = WriteMode::Overwrite;
|
params.mode = WriteMode::Overwrite;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if request.write_options.skip_auto_cleanup {
|
||||||
|
params.skip_auto_cleanup = true;
|
||||||
|
}
|
||||||
|
|
||||||
// Set up storage options if provided
|
// Set up storage options if provided
|
||||||
if let Some(storage_opts) = initial_storage_options {
|
if let Some(storage_opts) = initial_storage_options {
|
||||||
let store_params = params
|
let store_params = params
|
||||||
|
|||||||
@@ -234,6 +234,7 @@ mod test {
|
|||||||
.create_table("test", data)
|
.create_table("test", data)
|
||||||
.write_options(WriteOptions {
|
.write_options(WriteOptions {
|
||||||
lance_write_params: Some(param),
|
lance_write_params: Some(param),
|
||||||
|
..Default::default()
|
||||||
})
|
})
|
||||||
.execute()
|
.execute()
|
||||||
.await;
|
.await;
|
||||||
|
|||||||
@@ -189,6 +189,18 @@ pub struct WriteOptions {
|
|||||||
// Coming soon: https://github.com/lancedb/lancedb/issues/992
|
// Coming soon: https://github.com/lancedb/lancedb/issues/992
|
||||||
// /// What behavior to take if the data contains invalid vectors
|
// /// What behavior to take if the data contains invalid vectors
|
||||||
// pub on_bad_vectors: BadVectorHandling,
|
// pub on_bad_vectors: BadVectorHandling,
|
||||||
|
/// If true, skip the automatic cleanup of old dataset versions that would
|
||||||
|
/// otherwise run during the commit. This forwards to
|
||||||
|
/// [`WriteParams::skip_auto_cleanup`] in lance-core.
|
||||||
|
///
|
||||||
|
/// Useful for high-frequency writers that want to manage version cleanup
|
||||||
|
/// themselves (e.g. via a periodic optimize job), or for writers that
|
||||||
|
/// lack delete permissions on the underlying storage.
|
||||||
|
///
|
||||||
|
/// If `lance_write_params` is also set with `skip_auto_cleanup = true`,
|
||||||
|
/// the cleanup is skipped. Setting this field to `true` forces the flag
|
||||||
|
/// on regardless of `lance_write_params`.
|
||||||
|
pub skip_auto_cleanup: bool,
|
||||||
/// Advanced parameters that can be used to customize table creation
|
/// Advanced parameters that can be used to customize table creation
|
||||||
///
|
///
|
||||||
/// Overlapping `OpenTableBuilder` options (e.g. [AddDataBuilder::mode]) will take
|
/// Overlapping `OpenTableBuilder` options (e.g. [AddDataBuilder::mode]) will take
|
||||||
@@ -2283,7 +2295,8 @@ impl BaseTable for NativeTable {
|
|||||||
|
|
||||||
let output = add.into_plan(&table_schema, &table_def)?;
|
let output = add.into_plan(&table_schema, &table_def)?;
|
||||||
|
|
||||||
let lance_params = output
|
let skip_auto_cleanup = output.write_options.skip_auto_cleanup;
|
||||||
|
let mut lance_params = output
|
||||||
.write_options
|
.write_options
|
||||||
.lance_write_params
|
.lance_write_params
|
||||||
.unwrap_or(WriteParams {
|
.unwrap_or(WriteParams {
|
||||||
@@ -2293,6 +2306,9 @@ impl BaseTable for NativeTable {
|
|||||||
},
|
},
|
||||||
..Default::default()
|
..Default::default()
|
||||||
});
|
});
|
||||||
|
if skip_auto_cleanup {
|
||||||
|
lance_params.skip_auto_cleanup = true;
|
||||||
|
}
|
||||||
|
|
||||||
// Repartition for write parallelism if beneficial.
|
// Repartition for write parallelism if beneficial.
|
||||||
let plan = if num_partitions > 1 {
|
let plan = if num_partitions > 1 {
|
||||||
|
|||||||
@@ -441,6 +441,7 @@ mod tests {
|
|||||||
.add(new_batch.clone())
|
.add(new_batch.clone())
|
||||||
.write_options(WriteOptions {
|
.write_options(WriteOptions {
|
||||||
lance_write_params: Some(param),
|
lance_write_params: Some(param),
|
||||||
|
..Default::default()
|
||||||
})
|
})
|
||||||
.mode(AddDataMode::Append)
|
.mode(AddDataMode::Append)
|
||||||
.execute()
|
.execute()
|
||||||
@@ -761,4 +762,56 @@ mod tests {
|
|||||||
table2.add(struct_batch).execute().await.unwrap();
|
table2.add(struct_batch).execute().await.unwrap();
|
||||||
assert_eq!(table2.count_rows(None).await.unwrap(), 2);
|
assert_eq!(table2.count_rows(None).await.unwrap(), 2);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_add_skip_auto_cleanup() {
|
||||||
|
// Verifies WriteOptions::skip_auto_cleanup is forwarded to lance-core's
|
||||||
|
// WriteParams and actually suppresses the cleanup hook on commit.
|
||||||
|
let tmp_dir = tempfile::tempdir().unwrap();
|
||||||
|
let uri = tmp_dir.path().to_str().unwrap();
|
||||||
|
let conn = connect(uri).execute().await.unwrap();
|
||||||
|
|
||||||
|
let batch = record_batch!(("id", Int64, [1, 2, 3])).unwrap();
|
||||||
|
let table = conn.create_table("t", batch).execute().await.unwrap();
|
||||||
|
// Cleanup on every commit, with `older_than = 0s` so prior versions are
|
||||||
|
// immediately eligible.
|
||||||
|
table
|
||||||
|
.as_native()
|
||||||
|
.unwrap()
|
||||||
|
.update_config(vec![
|
||||||
|
("lance.auto_cleanup.interval".to_string(), "1".to_string()),
|
||||||
|
(
|
||||||
|
"lance.auto_cleanup.older_than".to_string(),
|
||||||
|
"0s".to_string(),
|
||||||
|
),
|
||||||
|
])
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// Write several versions with skip_auto_cleanup; none should be removed.
|
||||||
|
for i in 0..3 {
|
||||||
|
let new_batch = record_batch!(("id", Int64, [10 + i])).unwrap();
|
||||||
|
table
|
||||||
|
.add(new_batch)
|
||||||
|
.write_options(WriteOptions {
|
||||||
|
skip_auto_cleanup: true,
|
||||||
|
..Default::default()
|
||||||
|
})
|
||||||
|
.execute()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
let versions_before = table.list_versions().await.unwrap().len();
|
||||||
|
|
||||||
|
// Now write one more without the flag; cleanup should run and prune.
|
||||||
|
let new_batch = record_batch!(("id", Int64, [42])).unwrap();
|
||||||
|
table.add(new_batch).execute().await.unwrap();
|
||||||
|
let versions_after = table.list_versions().await.unwrap().len();
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
versions_after < versions_before,
|
||||||
|
"auto-cleanup should have removed old versions once the skip flag was off \
|
||||||
|
(before={versions_before}, after={versions_after})"
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -219,6 +219,7 @@ impl ExecutionPlan for InsertExec {
|
|||||||
&& let Some(merged_txn) = merge_transactions(transactions)
|
&& let Some(merged_txn) = merge_transactions(transactions)
|
||||||
{
|
{
|
||||||
let new_dataset = CommitBuilder::new(dataset.clone())
|
let new_dataset = CommitBuilder::new(dataset.clone())
|
||||||
|
.with_skip_auto_cleanup(write_params.skip_auto_cleanup)
|
||||||
.execute(merged_txn)
|
.execute(merged_txn)
|
||||||
.await?;
|
.await?;
|
||||||
ds_wrapper.update(new_dataset);
|
ds_wrapper.update(new_dataset);
|
||||||
|
|||||||
@@ -528,6 +528,7 @@ mod tests {
|
|||||||
}),
|
}),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
}),
|
}),
|
||||||
|
..Default::default()
|
||||||
})
|
})
|
||||||
.execute()
|
.execute()
|
||||||
.await
|
.await
|
||||||
@@ -589,6 +590,7 @@ mod tests {
|
|||||||
}),
|
}),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
}),
|
}),
|
||||||
|
..Default::default()
|
||||||
})
|
})
|
||||||
.execute()
|
.execute()
|
||||||
.await
|
.await
|
||||||
|
|||||||
@@ -55,6 +55,7 @@ pub struct MergeInsertBuilder {
|
|||||||
pub(crate) when_not_matched_by_source_delete_filt: Option<String>,
|
pub(crate) when_not_matched_by_source_delete_filt: Option<String>,
|
||||||
pub(crate) timeout: Option<Duration>,
|
pub(crate) timeout: Option<Duration>,
|
||||||
pub(crate) use_index: bool,
|
pub(crate) use_index: bool,
|
||||||
|
pub(crate) skip_auto_cleanup: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl MergeInsertBuilder {
|
impl MergeInsertBuilder {
|
||||||
@@ -69,6 +70,7 @@ impl MergeInsertBuilder {
|
|||||||
when_not_matched_by_source_delete_filt: None,
|
when_not_matched_by_source_delete_filt: None,
|
||||||
timeout: None,
|
timeout: None,
|
||||||
use_index: true,
|
use_index: true,
|
||||||
|
skip_auto_cleanup: false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -148,6 +150,17 @@ impl MergeInsertBuilder {
|
|||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Skip the automatic cleanup of old dataset versions that would otherwise
|
||||||
|
/// run during the merge insert commit.
|
||||||
|
///
|
||||||
|
/// This forwards to [`lance::dataset::MergeInsertBuilder::skip_auto_cleanup`]
|
||||||
|
/// in lance-core. Useful for high-frequency writers that want to manage
|
||||||
|
/// version cleanup themselves, or writers without delete permissions.
|
||||||
|
pub fn skip_auto_cleanup(&mut self, skip: bool) -> &mut Self {
|
||||||
|
self.skip_auto_cleanup = skip;
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
/// Executes the merge insert operation
|
/// Executes the merge insert operation
|
||||||
///
|
///
|
||||||
/// Returns version and statistics about the merge operation including the number of rows
|
/// Returns version and statistics about the merge operation including the number of rows
|
||||||
@@ -191,6 +204,9 @@ pub(crate) async fn execute_merge_insert(
|
|||||||
builder.when_not_matched_by_source(WhenNotMatchedBySource::Keep);
|
builder.when_not_matched_by_source(WhenNotMatchedBySource::Keep);
|
||||||
}
|
}
|
||||||
builder.use_index(params.use_index);
|
builder.use_index(params.use_index);
|
||||||
|
if params.skip_auto_cleanup {
|
||||||
|
builder.skip_auto_cleanup(true);
|
||||||
|
}
|
||||||
|
|
||||||
let future = if let Some(timeout) = params.timeout {
|
let future = if let Some(timeout) = params.timeout {
|
||||||
let future = builder
|
let future = builder
|
||||||
|
|||||||
Reference in New Issue
Block a user