mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-29 09:28:27 +00:00
Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 32c77879c9 | |||
| 9330a9b851 | |||
| 02de07576e | |||
| 81617fd3d9 |
@@ -45,7 +45,9 @@ jobs:
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 20
|
||||
# pnpm 11 (used by the nodejs install step below) requires
|
||||
# Node >= 22.13; use 24 since 22 hits EOL in October.
|
||||
node-version: 24
|
||||
|
||||
- name: Install Codex CLI
|
||||
run: npm install -g @openai/codex
|
||||
@@ -79,10 +81,14 @@ jobs:
|
||||
java-version: '11'
|
||||
cache: maven
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v4
|
||||
with:
|
||||
version: 11.1.1
|
||||
- name: Install Node.js dependencies for TypeScript bindings
|
||||
run: |
|
||||
cd nodejs
|
||||
npm ci
|
||||
pnpm install --frozen-lockfile
|
||||
|
||||
- name: Configure git user
|
||||
run: |
|
||||
@@ -137,7 +143,7 @@ jobs:
|
||||
- For Rust test failures: Run the specific test with "cargo test -p <crate> <test_name>"
|
||||
- For Python test failures: Build with "cd python && maturin develop" then run "pytest <specific_test_file>::<test_name>"
|
||||
- For Java test failures: Run "cd java && mvn test -Dtest=<TestClass>#<testMethod>"
|
||||
- For TypeScript test failures: Run "cd nodejs && npm run build && npm test -- --testNamePattern='<test_name>'"
|
||||
- For TypeScript test failures: Run "cd nodejs && pnpm build && pnpm test -- --testNamePattern='<test_name>'"
|
||||
- Do NOT run the full test suite - only run the tests that were failing
|
||||
|
||||
7. If the additional guidelines are provided, follow them as well.
|
||||
|
||||
@@ -42,11 +42,17 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
lfs: true
|
||||
- uses: pnpm/action-setup@v4
|
||||
with:
|
||||
version: 11.1.1
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 20
|
||||
cache: 'npm'
|
||||
cache-dependency-path: nodejs/package-lock.json
|
||||
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
|
||||
# in October. The library itself still supports Node >= 18
|
||||
# (see test matrix below).
|
||||
node-version: 24
|
||||
cache: 'pnpm'
|
||||
cache-dependency-path: nodejs/pnpm-lock.yaml
|
||||
- uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
with:
|
||||
components: rustfmt, clippy
|
||||
@@ -61,11 +67,13 @@ jobs:
|
||||
run: cargo clippy --profile ci --all --all-features -- -D warnings
|
||||
- name: Lint Typescript
|
||||
run: |
|
||||
npm ci
|
||||
npm run lint-ci
|
||||
pnpm install --frozen-lockfile
|
||||
pnpm lint-ci
|
||||
- name: Lint examples
|
||||
working-directory: nodejs/examples
|
||||
run: npm ci && npm run lint-ci
|
||||
# The `@lancedb/lancedb` dep points at file:../dist; pnpm errors if
|
||||
# that dir is missing, so create an empty one for lint-only runs.
|
||||
run: mkdir -p ../dist && pnpm install --frozen-lockfile && pnpm lint-ci
|
||||
linux:
|
||||
name: Linux (NodeJS ${{ matrix.node-version }})
|
||||
timeout-minutes: 30
|
||||
@@ -82,14 +90,18 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
lfs: true
|
||||
- uses: actions/setup-node@v4
|
||||
name: Setup Node.js 20 for build
|
||||
- uses: pnpm/action-setup@v4
|
||||
with:
|
||||
# @napi-rs/cli v3 requires Node >= 20.12 (via @inquirer/prompts@8).
|
||||
# Build always on Node 20; tests run on the matrix version below.
|
||||
node-version: 20
|
||||
cache: 'npm'
|
||||
cache-dependency-path: nodejs/package-lock.json
|
||||
version: 11.1.1
|
||||
- uses: actions/setup-node@v4
|
||||
name: Setup Node.js 24 for build
|
||||
with:
|
||||
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
|
||||
# in October. Build/install runs on Node 24; tests run on the
|
||||
# matrix version below using direct jest invocation.
|
||||
node-version: 24
|
||||
cache: 'pnpm'
|
||||
cache-dependency-path: nodejs/pnpm-lock.yaml
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
@@ -97,45 +109,52 @@ jobs:
|
||||
sudo apt install -y protobuf-compiler libssl-dev
|
||||
- name: Build
|
||||
run: |
|
||||
npm ci --include=optional
|
||||
npm run build:debug -- --profile ci
|
||||
pnpm install --frozen-lockfile
|
||||
# No `--` separator: pnpm forwards it literally, which would
|
||||
# make napi-rs treat `--profile ci` as a cargo passthrough arg.
|
||||
pnpm build:debug --profile ci
|
||||
pnpm tsc
|
||||
- name: Setup examples
|
||||
working-directory: nodejs/examples
|
||||
run: pnpm install --frozen-lockfile
|
||||
- name: Check docs
|
||||
run: |
|
||||
# We run this as part of the job because the binary needs to be built
|
||||
# first to export the types of the native code.
|
||||
set -e
|
||||
# `pnpm docs` would invoke pnpm's built-in `docs` command, not
|
||||
# the script — use `pnpm run docs`.
|
||||
pnpm run docs
|
||||
if ! git diff --exit-code -- ../ ':(exclude)Cargo.lock'; then
|
||||
echo "Docs need to be updated"
|
||||
echo "Run 'pnpm run docs', fix any warnings, and commit the changes."
|
||||
exit 1
|
||||
fi
|
||||
- uses: actions/setup-node@v4
|
||||
name: Setup Node.js ${{ matrix.node-version }} for test
|
||||
with:
|
||||
node-version: ${{ matrix.node-version }}
|
||||
- name: Compile TypeScript
|
||||
run: npm run tsc
|
||||
- name: Setup localstack
|
||||
working-directory: .
|
||||
run: docker compose up --detach --wait
|
||||
- name: Test
|
||||
env:
|
||||
S3_TEST: "1"
|
||||
run: npm run test
|
||||
- name: Setup examples
|
||||
working-directory: nodejs/examples
|
||||
run: npm ci
|
||||
# Newer @smithy/core uses dynamic ESM imports.
|
||||
NODE_OPTIONS: "--experimental-vm-modules"
|
||||
# Invoke jest directly because pnpm 11 itself requires Node 22+
|
||||
# while the matrix tests on older Node versions.
|
||||
run: npx jest --verbose
|
||||
- name: Test examples
|
||||
working-directory: ./
|
||||
env:
|
||||
OPENAI_API_KEY: test
|
||||
OPENAI_BASE_URL: http://0.0.0.0:8000
|
||||
NODE_OPTIONS: "--experimental-vm-modules"
|
||||
run: |
|
||||
python ci/mock_openai.py &
|
||||
cd nodejs/examples
|
||||
npm test
|
||||
- name: Check docs
|
||||
run: |
|
||||
# We run this as part of the job because the binary needs to be built
|
||||
# first to export the types of the native code.
|
||||
set -e
|
||||
npm ci
|
||||
npm run docs
|
||||
if ! git diff --exit-code -- ../ ':(exclude)Cargo.lock'; then
|
||||
echo "Docs need to be updated"
|
||||
echo "Run 'npm run docs', fix any warnings, and commit the changes."
|
||||
exit 1
|
||||
fi
|
||||
npx jest --testEnvironment jest-environment-node-single-context --verbose
|
||||
macos:
|
||||
timeout-minutes: 30
|
||||
runs-on: "macos-14"
|
||||
@@ -148,20 +167,28 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
lfs: true
|
||||
- uses: pnpm/action-setup@v4
|
||||
with:
|
||||
version: 11.1.1
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 20
|
||||
cache: 'npm'
|
||||
cache-dependency-path: nodejs/package-lock.json
|
||||
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
|
||||
# in October.
|
||||
node-version: 24
|
||||
cache: 'pnpm'
|
||||
cache-dependency-path: nodejs/pnpm-lock.yaml
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
brew install protobuf
|
||||
- name: Build
|
||||
run: |
|
||||
npm ci --include=optional
|
||||
npm run build:debug -- --profile ci
|
||||
npm run tsc
|
||||
pnpm install --frozen-lockfile
|
||||
# No `--` separator: pnpm forwards it literally, which would
|
||||
# make napi-rs treat `--profile ci` as a cargo passthrough arg.
|
||||
pnpm build:debug --profile ci
|
||||
pnpm tsc
|
||||
- name: Test
|
||||
run: |
|
||||
npm run test
|
||||
pnpm test
|
||||
|
||||
@@ -171,13 +171,18 @@ jobs:
|
||||
working-directory: nodejs
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v4
|
||||
with:
|
||||
version: 11.1.1
|
||||
- name: Setup node
|
||||
uses: actions/setup-node@v4
|
||||
if: ${{ !matrix.settings.docker }}
|
||||
with:
|
||||
node-version: 20
|
||||
cache: npm
|
||||
cache-dependency-path: nodejs/package-lock.json
|
||||
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
|
||||
# in October.
|
||||
node-version: 24
|
||||
cache: pnpm
|
||||
cache-dependency-path: nodejs/pnpm-lock.yaml
|
||||
- name: Install
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
if: ${{ !matrix.settings.docker }}
|
||||
@@ -195,7 +200,7 @@ jobs:
|
||||
target/
|
||||
key: nodejs-${{ matrix.settings.target }}-cargo-${{ matrix.settings.host }}
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
run: pnpm install --frozen-lockfile
|
||||
- name: Install Zig
|
||||
uses: mlugg/setup-zig@v2
|
||||
if: ${{ contains(matrix.settings.target, 'musl') }}
|
||||
@@ -248,7 +253,7 @@ jobs:
|
||||
# one to do the upload.
|
||||
- name: Make generic artifacts
|
||||
if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }}
|
||||
run: npm run tsc
|
||||
run: pnpm tsc
|
||||
- name: Upload Generic Artifacts
|
||||
if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }}
|
||||
uses: actions/upload-artifact@v4
|
||||
@@ -283,14 +288,24 @@ jobs:
|
||||
working-directory: nodejs
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Setup node
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v4
|
||||
with:
|
||||
version: 11.1.1
|
||||
- name: Setup Node.js 24 for install
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
|
||||
# in October.
|
||||
node-version: 24
|
||||
cache: pnpm
|
||||
cache-dependency-path: nodejs/pnpm-lock.yaml
|
||||
- name: Install dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
- name: Setup Node.js ${{ matrix.node }} for test
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: ${{ matrix.node }}
|
||||
cache: npm
|
||||
cache-dependency-path: nodejs/package-lock.json
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
- name: Download artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
@@ -311,7 +326,9 @@ jobs:
|
||||
- name: Move built files
|
||||
run: cp dist/native.d.ts dist/native.js dist/*.node lancedb/
|
||||
- name: Test bindings
|
||||
run: npm test
|
||||
# Invoke jest directly because pnpm 11 itself requires Node 22+
|
||||
# while the matrix tests on older Node versions.
|
||||
run: npx jest --verbose
|
||||
publish:
|
||||
name: Publish
|
||||
runs-on: ubuntu-latest
|
||||
@@ -323,15 +340,19 @@ jobs:
|
||||
- test-lancedb
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v4
|
||||
with:
|
||||
version: 11.1.1
|
||||
- name: Setup node
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
cache: npm
|
||||
cache-dependency-path: nodejs/package-lock.json
|
||||
cache: pnpm
|
||||
cache-dependency-path: nodejs/pnpm-lock.yaml
|
||||
registry-url: "https://registry.npmjs.org"
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
run: pnpm install --frozen-lockfile
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: nodejs-dist
|
||||
@@ -351,7 +372,7 @@ jobs:
|
||||
- name: Display structure of downloaded files
|
||||
run: find dist && find nodejs-artifacts
|
||||
- name: Move artifacts
|
||||
run: npx napi artifacts -d nodejs-artifacts
|
||||
run: pnpm exec napi artifacts -d nodejs-artifacts
|
||||
- name: List packages
|
||||
run: find npm
|
||||
- name: Publish
|
||||
|
||||
Generated
+1
@@ -5099,6 +5099,7 @@ dependencies = [
|
||||
"env_logger",
|
||||
"futures",
|
||||
"half",
|
||||
"lance-namespace",
|
||||
"lancedb",
|
||||
"log",
|
||||
"lzma-sys",
|
||||
|
||||
@@ -12,20 +12,22 @@ Typescript.
|
||||
* `src/`: Rust bindings source code
|
||||
* `lancedb/`: Typescript package source code
|
||||
* `__test__/`: Unit tests
|
||||
* `examples/`: An npm package with the examples shown in the documentation
|
||||
* `examples/`: A pnpm package with the examples shown in the documentation
|
||||
|
||||
## Development environment
|
||||
|
||||
To set up your development environment, you will need to install the following:
|
||||
|
||||
1. Node.js 14 or later
|
||||
2. Rust's package manager, Cargo. Use [rustup](https://rustup.rs/) to install.
|
||||
3. [protoc](https://grpc.io/docs/protoc-installation/) (Protocol Buffers compiler)
|
||||
1. Node.js 22 or later (required by pnpm 11)
|
||||
2. [pnpm](https://pnpm.io/installation) 11 or later (or run via `corepack enable`,
|
||||
which uses the `packageManager` field in `package.json`)
|
||||
3. Rust's package manager, Cargo. Use [rustup](https://rustup.rs/) to install.
|
||||
4. [protoc](https://grpc.io/docs/protoc-installation/) (Protocol Buffers compiler)
|
||||
|
||||
Initial setup:
|
||||
|
||||
```shell
|
||||
npm install
|
||||
pnpm install
|
||||
```
|
||||
|
||||
### Commit Hooks
|
||||
@@ -39,38 +41,38 @@ pre-commit install
|
||||
|
||||
## Development
|
||||
|
||||
Most common development commands can be run using the npm scripts.
|
||||
Most common development commands can be run using the pnpm scripts.
|
||||
|
||||
Build the package
|
||||
|
||||
```shell
|
||||
npm install
|
||||
npm run build
|
||||
pnpm install
|
||||
pnpm build
|
||||
```
|
||||
|
||||
Lint:
|
||||
|
||||
```shell
|
||||
npm run lint
|
||||
pnpm lint
|
||||
```
|
||||
|
||||
Format and fix lints:
|
||||
|
||||
```shell
|
||||
npm run lint-fix
|
||||
pnpm lint-fix
|
||||
```
|
||||
|
||||
Run tests:
|
||||
|
||||
```shell
|
||||
npm test
|
||||
pnpm test
|
||||
```
|
||||
|
||||
To run a single test:
|
||||
|
||||
```shell
|
||||
# Single file: table.test.ts
|
||||
npm test -- table.test.ts
|
||||
pnpm test -- table.test.ts
|
||||
# Single test: 'merge insert' in table.test.ts
|
||||
npm test -- table.test.ts --testNamePattern=merge\ insert
|
||||
pnpm test -- table.test.ts --testNamePattern=merge\ insert
|
||||
```
|
||||
|
||||
@@ -148,6 +148,33 @@ Creates a new empty Table
|
||||
|
||||
***
|
||||
|
||||
### createNamespace()
|
||||
|
||||
```ts
|
||||
abstract createNamespace(namespacePath, options?): Promise<CreateNamespaceResponse>
|
||||
```
|
||||
|
||||
Create a new namespace at the given path.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **namespacePath**: `string`[]
|
||||
The namespace path to create.
|
||||
|
||||
* **options?**: `Partial`<[`CreateNamespaceOptions`](../interfaces/CreateNamespaceOptions.md)>
|
||||
Creation `mode`
|
||||
("create" | "exist_ok" | "overwrite") and optional `properties`
|
||||
to attach to the namespace.
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<[`CreateNamespaceResponse`](../interfaces/CreateNamespaceResponse.md)>
|
||||
|
||||
The properties of the
|
||||
created namespace and an optional transaction id.
|
||||
|
||||
***
|
||||
|
||||
### createTable()
|
||||
|
||||
#### createTable(options, namespacePath)
|
||||
@@ -230,6 +257,29 @@ Creates a new Table and initialize it with new data.
|
||||
|
||||
***
|
||||
|
||||
### describeNamespace()
|
||||
|
||||
```ts
|
||||
abstract describeNamespace(namespacePath): Promise<DescribeNamespaceResponse>
|
||||
```
|
||||
|
||||
Describe a namespace, returning its properties.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **namespacePath**: `string`[]
|
||||
The namespace path to describe, in
|
||||
parent → child order, e.g. `["analytics", "sales"]`.
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<[`DescribeNamespaceResponse`](../interfaces/DescribeNamespaceResponse.md)>
|
||||
|
||||
The namespace's properties
|
||||
(may be undefined if the namespace has none).
|
||||
|
||||
***
|
||||
|
||||
### display()
|
||||
|
||||
```ts
|
||||
@@ -263,6 +313,36 @@ Drop all tables in the database.
|
||||
|
||||
***
|
||||
|
||||
### dropNamespace()
|
||||
|
||||
```ts
|
||||
abstract dropNamespace(namespacePath, options?): Promise<DropNamespaceResponse>
|
||||
```
|
||||
|
||||
Drop a namespace.
|
||||
|
||||
Use `behavior: "cascade"` to also drop everything contained in the
|
||||
namespace (sub-namespaces and tables). The default `"restrict"`
|
||||
behavior refuses to drop a non-empty namespace.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **namespacePath**: `string`[]
|
||||
The namespace path to drop.
|
||||
|
||||
* **options?**: `Partial`<[`DropNamespaceOptions`](../interfaces/DropNamespaceOptions.md)>
|
||||
`mode` ("skip" | "fail"
|
||||
for missing-namespace handling) and `behavior` ("restrict" | "cascade").
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<[`DropNamespaceResponse`](../interfaces/DropNamespaceResponse.md)>
|
||||
|
||||
Any properties returned by
|
||||
the server and an optional transaction id.
|
||||
|
||||
***
|
||||
|
||||
### dropTable()
|
||||
|
||||
```ts
|
||||
@@ -299,6 +379,36 @@ Return true if the connection has not been closed
|
||||
|
||||
***
|
||||
|
||||
### listNamespaces()
|
||||
|
||||
```ts
|
||||
abstract listNamespaces(namespacePath?, options?): Promise<ListNamespacesResponse>
|
||||
```
|
||||
|
||||
List the immediate child namespaces under the given parent.
|
||||
|
||||
Results may be paginated. To retrieve subsequent pages, pass the
|
||||
`pageToken` returned by a previous call.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **namespacePath?**: `string`[]
|
||||
The parent namespace path. Defaults
|
||||
to the root namespace if omitted.
|
||||
|
||||
* **options?**: `Partial`<[`ListNamespacesOptions`](../interfaces/ListNamespacesOptions.md)>
|
||||
Pagination options
|
||||
(`pageToken`, `limit`).
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<[`ListNamespacesResponse`](../interfaces/ListNamespacesResponse.md)>
|
||||
|
||||
Child namespace names and
|
||||
an optional token for fetching the next page.
|
||||
|
||||
***
|
||||
|
||||
### openTable()
|
||||
|
||||
```ts
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / connectNamespace
|
||||
|
||||
# Function: connectNamespace()
|
||||
|
||||
## connectNamespace(implName, config, options)
|
||||
|
||||
```ts
|
||||
function connectNamespace(
|
||||
implName,
|
||||
config,
|
||||
options?): Promise<Connection>
|
||||
```
|
||||
|
||||
Connect to a LanceDB database through a namespace.
|
||||
|
||||
Unlike [connect](connect.md), which routes by URI scheme (local path vs.
|
||||
`db://` cloud), `connectNamespace` always returns a namespace-backed
|
||||
connection. The `implName` selects the namespace implementation:
|
||||
|
||||
- `"dir"` — directory namespace, configured with [DirNamespaceConfig](../interfaces/DirNamespaceConfig.md).
|
||||
- `"rest"` — remote REST catalog, configured with [RestNamespaceConfig](../interfaces/RestNamespaceConfig.md).
|
||||
- Any other string — full module path for a custom implementation,
|
||||
configured with a free-form string-keyed `properties` map.
|
||||
|
||||
### Parameters
|
||||
|
||||
* **implName**: `"dir"`
|
||||
|
||||
* **config**: [`DirNamespaceConfig`](../interfaces/DirNamespaceConfig.md)
|
||||
|
||||
* **options?**: `Partial`<[`ConnectNamespaceOptions`](../interfaces/ConnectNamespaceOptions.md)>
|
||||
|
||||
### Returns
|
||||
|
||||
`Promise`<[`Connection`](../classes/Connection.md)>
|
||||
|
||||
### Examples
|
||||
|
||||
```ts
|
||||
const db = await connectNamespace("dir", { root: "/path/to/db" });
|
||||
await db.createTable("users", [{ id: 1 }]);
|
||||
```
|
||||
|
||||
```ts
|
||||
const db = await connectNamespace("rest", {
|
||||
uri: "https://catalog.example.com",
|
||||
headers: { "x-api-key": process.env.CATALOG_KEY ?? "" },
|
||||
});
|
||||
```
|
||||
|
||||
```ts
|
||||
const db = await connectNamespace("my.custom.Namespace", {
|
||||
endpoint: "...",
|
||||
});
|
||||
```
|
||||
|
||||
## connectNamespace(implName, config, options)
|
||||
|
||||
```ts
|
||||
function connectNamespace(
|
||||
implName,
|
||||
config,
|
||||
options?): Promise<Connection>
|
||||
```
|
||||
|
||||
Connect through the built-in REST namespace.
|
||||
|
||||
Configured with [RestNamespaceConfig](../interfaces/RestNamespaceConfig.md). See the function-level
|
||||
documentation above for the full surface, examples, and how this
|
||||
relates to [connect](connect.md).
|
||||
|
||||
### Parameters
|
||||
|
||||
* **implName**: `"rest"`
|
||||
|
||||
* **config**: [`RestNamespaceConfig`](../interfaces/RestNamespaceConfig.md)
|
||||
|
||||
* **options?**: `Partial`<[`ConnectNamespaceOptions`](../interfaces/ConnectNamespaceOptions.md)>
|
||||
|
||||
### Returns
|
||||
|
||||
`Promise`<[`Connection`](../classes/Connection.md)>
|
||||
|
||||
### Example
|
||||
|
||||
```ts
|
||||
const db = await connectNamespace("rest", {
|
||||
uri: "https://catalog.example.com",
|
||||
headers: { "x-api-key": process.env.CATALOG_KEY ?? "" },
|
||||
});
|
||||
```
|
||||
|
||||
## connectNamespace(implName, properties, options)
|
||||
|
||||
```ts
|
||||
function connectNamespace(
|
||||
implName,
|
||||
properties,
|
||||
options?): Promise<Connection>
|
||||
```
|
||||
|
||||
Connect through a custom namespace implementation by full module path,
|
||||
configured with a free-form string-keyed `properties` map. Use the
|
||||
typed overloads above for the built-in `"dir"` and `"rest"` impls.
|
||||
|
||||
See the function-level documentation above for examples and how this
|
||||
relates to [connect](connect.md).
|
||||
|
||||
### Parameters
|
||||
|
||||
* **implName**: `string`
|
||||
|
||||
* **properties**: `Record`<`string`, `string`>
|
||||
|
||||
* **options?**: `Partial`<[`ConnectNamespaceOptions`](../interfaces/ConnectNamespaceOptions.md)>
|
||||
|
||||
### Returns
|
||||
|
||||
`Promise`<[`Connection`](../classes/Connection.md)>
|
||||
|
||||
### Example
|
||||
|
||||
```ts
|
||||
const db = await connectNamespace("my.custom.Namespace", {
|
||||
endpoint: "...",
|
||||
});
|
||||
```
|
||||
@@ -51,10 +51,17 @@
|
||||
- [ClientConfig](interfaces/ClientConfig.md)
|
||||
- [ColumnAlteration](interfaces/ColumnAlteration.md)
|
||||
- [CompactionStats](interfaces/CompactionStats.md)
|
||||
- [ConnectNamespaceOptions](interfaces/ConnectNamespaceOptions.md)
|
||||
- [ConnectionOptions](interfaces/ConnectionOptions.md)
|
||||
- [CreateNamespaceOptions](interfaces/CreateNamespaceOptions.md)
|
||||
- [CreateNamespaceResponse](interfaces/CreateNamespaceResponse.md)
|
||||
- [CreateTableOptions](interfaces/CreateTableOptions.md)
|
||||
- [DeleteResult](interfaces/DeleteResult.md)
|
||||
- [DescribeNamespaceResponse](interfaces/DescribeNamespaceResponse.md)
|
||||
- [DirNamespaceConfig](interfaces/DirNamespaceConfig.md)
|
||||
- [DropColumnsResult](interfaces/DropColumnsResult.md)
|
||||
- [DropNamespaceOptions](interfaces/DropNamespaceOptions.md)
|
||||
- [DropNamespaceResponse](interfaces/DropNamespaceResponse.md)
|
||||
- [ExecutableQuery](interfaces/ExecutableQuery.md)
|
||||
- [FragmentStatistics](interfaces/FragmentStatistics.md)
|
||||
- [FragmentSummaryStats](interfaces/FragmentSummaryStats.md)
|
||||
@@ -69,12 +76,15 @@
|
||||
- [IvfFlatOptions](interfaces/IvfFlatOptions.md)
|
||||
- [IvfPqOptions](interfaces/IvfPqOptions.md)
|
||||
- [IvfRqOptions](interfaces/IvfRqOptions.md)
|
||||
- [ListNamespacesOptions](interfaces/ListNamespacesOptions.md)
|
||||
- [ListNamespacesResponse](interfaces/ListNamespacesResponse.md)
|
||||
- [MergeResult](interfaces/MergeResult.md)
|
||||
- [OpenTableOptions](interfaces/OpenTableOptions.md)
|
||||
- [OptimizeOptions](interfaces/OptimizeOptions.md)
|
||||
- [OptimizeStats](interfaces/OptimizeStats.md)
|
||||
- [QueryExecutionOptions](interfaces/QueryExecutionOptions.md)
|
||||
- [RemovalStats](interfaces/RemovalStats.md)
|
||||
- [RestNamespaceConfig](interfaces/RestNamespaceConfig.md)
|
||||
- [RetryConfig](interfaces/RetryConfig.md)
|
||||
- [ShuffleOptions](interfaces/ShuffleOptions.md)
|
||||
- [SplitCalculatedOptions](interfaces/SplitCalculatedOptions.md)
|
||||
@@ -107,6 +117,7 @@
|
||||
|
||||
- [RecordBatchIterator](functions/RecordBatchIterator.md)
|
||||
- [connect](functions/connect.md)
|
||||
- [connectNamespace](functions/connectNamespace.md)
|
||||
- [makeArrowTable](functions/makeArrowTable.md)
|
||||
- [packBits](functions/packBits.md)
|
||||
- [permutationBuilder](functions/permutationBuilder.md)
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / ConnectNamespaceOptions
|
||||
|
||||
# Interface: ConnectNamespaceOptions
|
||||
|
||||
## Properties
|
||||
|
||||
### namespaceClientProperties?
|
||||
|
||||
```ts
|
||||
optional namespaceClientProperties: Record<string, string>;
|
||||
```
|
||||
|
||||
Extra properties for the backing namespace client.
|
||||
|
||||
***
|
||||
|
||||
### readConsistencyInterval?
|
||||
|
||||
```ts
|
||||
optional readConsistencyInterval: number;
|
||||
```
|
||||
|
||||
The interval, in seconds, at which to check for updates to the table
|
||||
from other processes. If None, then consistency is not checked. For
|
||||
performance reasons, this is the default. For strong consistency, set
|
||||
this to zero seconds. Then every read will check for updates from other
|
||||
processes. As a compromise, you can set this to a non-zero value for
|
||||
eventual consistency.
|
||||
|
||||
***
|
||||
|
||||
### session?
|
||||
|
||||
```ts
|
||||
optional session: Session;
|
||||
```
|
||||
|
||||
The session to use for this connection. Holds shared caches and other
|
||||
session-specific state.
|
||||
|
||||
***
|
||||
|
||||
### storageOptions?
|
||||
|
||||
```ts
|
||||
optional storageOptions: Record<string, string>;
|
||||
```
|
||||
|
||||
Configuration for object storage. The available options are described
|
||||
at https://docs.lancedb.com/storage/
|
||||
@@ -0,0 +1,27 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / CreateNamespaceOptions
|
||||
|
||||
# Interface: CreateNamespaceOptions
|
||||
|
||||
## Properties
|
||||
|
||||
### mode?
|
||||
|
||||
```ts
|
||||
optional mode: "overwrite" | "create" | "exist_ok";
|
||||
```
|
||||
|
||||
Creation mode.
|
||||
|
||||
***
|
||||
|
||||
### properties?
|
||||
|
||||
```ts
|
||||
optional properties: Record<string, string>;
|
||||
```
|
||||
|
||||
Properties to set on the new namespace.
|
||||
@@ -0,0 +1,23 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / CreateNamespaceResponse
|
||||
|
||||
# Interface: CreateNamespaceResponse
|
||||
|
||||
## Properties
|
||||
|
||||
### properties?
|
||||
|
||||
```ts
|
||||
optional properties: Record<string, string>;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### transactionId?
|
||||
|
||||
```ts
|
||||
optional transactionId: string;
|
||||
```
|
||||
@@ -0,0 +1,15 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / DescribeNamespaceResponse
|
||||
|
||||
# Interface: DescribeNamespaceResponse
|
||||
|
||||
## Properties
|
||||
|
||||
### properties?
|
||||
|
||||
```ts
|
||||
optional properties: Record<string, string>;
|
||||
```
|
||||
@@ -0,0 +1,47 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / DirNamespaceConfig
|
||||
|
||||
# Interface: DirNamespaceConfig
|
||||
|
||||
Configuration for the built-in directory namespace (`"dir"`).
|
||||
|
||||
The directory namespace stores tables under a single root path (local
|
||||
filesystem or object storage URI). See
|
||||
[https://docs.lancedb.com/namespaces](https://docs.lancedb.com/namespaces) for the documented surface;
|
||||
less-common knobs live under [DirNamespaceConfig.extraProperties](DirNamespaceConfig.md#extraproperties).
|
||||
|
||||
## Properties
|
||||
|
||||
### extraProperties?
|
||||
|
||||
```ts
|
||||
optional extraProperties: Record<string, string>;
|
||||
```
|
||||
|
||||
Additional raw properties passed verbatim to the namespace
|
||||
implementation (e.g. `storage.*`, `credential_vendor.*`). Typed
|
||||
fields above take precedence on key collision.
|
||||
|
||||
***
|
||||
|
||||
### manifestEnabled?
|
||||
|
||||
```ts
|
||||
optional manifestEnabled: boolean;
|
||||
```
|
||||
|
||||
Whether to maintain a namespace manifest at the root. Required for
|
||||
child namespaces. Defaults to true on the impl side.
|
||||
|
||||
***
|
||||
|
||||
### root
|
||||
|
||||
```ts
|
||||
root: string;
|
||||
```
|
||||
|
||||
Root path or URI containing the LanceDB tables.
|
||||
@@ -0,0 +1,27 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / DropNamespaceOptions
|
||||
|
||||
# Interface: DropNamespaceOptions
|
||||
|
||||
## Properties
|
||||
|
||||
### behavior?
|
||||
|
||||
```ts
|
||||
optional behavior: "restrict" | "cascade";
|
||||
```
|
||||
|
||||
Refuse to drop if non-empty (restrict) or drop recursively (cascade).
|
||||
|
||||
***
|
||||
|
||||
### mode?
|
||||
|
||||
```ts
|
||||
optional mode: "fail" | "skip";
|
||||
```
|
||||
|
||||
Whether to skip if the namespace doesn't exist, or fail.
|
||||
@@ -0,0 +1,23 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / DropNamespaceResponse
|
||||
|
||||
# Interface: DropNamespaceResponse
|
||||
|
||||
## Properties
|
||||
|
||||
### properties?
|
||||
|
||||
```ts
|
||||
optional properties: Record<string, string>;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### transactionId?
|
||||
|
||||
```ts
|
||||
optional transactionId: string[];
|
||||
```
|
||||
@@ -0,0 +1,27 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / ListNamespacesOptions
|
||||
|
||||
# Interface: ListNamespacesOptions
|
||||
|
||||
## Properties
|
||||
|
||||
### limit?
|
||||
|
||||
```ts
|
||||
optional limit: number;
|
||||
```
|
||||
|
||||
An optional limit to the number of results to return.
|
||||
|
||||
***
|
||||
|
||||
### pageToken?
|
||||
|
||||
```ts
|
||||
optional pageToken: string;
|
||||
```
|
||||
|
||||
Token from a previous response for pagination.
|
||||
@@ -0,0 +1,23 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / ListNamespacesResponse
|
||||
|
||||
# Interface: ListNamespacesResponse
|
||||
|
||||
## Properties
|
||||
|
||||
### namespaces
|
||||
|
||||
```ts
|
||||
namespaces: string[];
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### pageToken?
|
||||
|
||||
```ts
|
||||
optional pageToken: string;
|
||||
```
|
||||
@@ -0,0 +1,47 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / RestNamespaceConfig
|
||||
|
||||
# Interface: RestNamespaceConfig
|
||||
|
||||
Configuration for the built-in REST namespace (`"rest"`).
|
||||
|
||||
The REST namespace talks to a remote catalog server over HTTP. See
|
||||
[https://docs.lancedb.com/namespaces](https://docs.lancedb.com/namespaces) for the documented surface;
|
||||
less-common knobs (TLS, metrics) live under
|
||||
[RestNamespaceConfig.extraProperties](RestNamespaceConfig.md#extraproperties).
|
||||
|
||||
## Properties
|
||||
|
||||
### extraProperties?
|
||||
|
||||
```ts
|
||||
optional extraProperties: Record<string, string>;
|
||||
```
|
||||
|
||||
Additional raw properties passed verbatim to the namespace
|
||||
implementation (e.g. `tls.*`, `ops_metrics_enabled`, `delimiter`).
|
||||
Typed fields above take precedence on key collision.
|
||||
|
||||
***
|
||||
|
||||
### headers?
|
||||
|
||||
```ts
|
||||
optional headers: Record<string, string>;
|
||||
```
|
||||
|
||||
HTTP headers forwarded with each request. Keys are passed through
|
||||
as-is (e.g. `"x-api-key"`, `"Authorization"`).
|
||||
|
||||
***
|
||||
|
||||
### uri
|
||||
|
||||
```ts
|
||||
uri: string;
|
||||
```
|
||||
|
||||
Catalog endpoint URL.
|
||||
+6
-6
@@ -3,11 +3,11 @@ The core Rust library is in the `../rust/lancedb` directory, the rust binding
|
||||
code is in the `src/` directory and the typescript bindings are in
|
||||
the `lancedb/` directory.
|
||||
|
||||
Whenever you change the Rust code, you will need to recompile: `npm run build`.
|
||||
Whenever you change the Rust code, you will need to recompile: `pnpm build`.
|
||||
|
||||
Common commands:
|
||||
* Build: `npm run build`
|
||||
* Lint: `npm run lint`
|
||||
* Fix lints: `npm run lint-fix`
|
||||
* Test: `npm test`
|
||||
* Run single test file: `npm test __test__/arrow.test.ts`
|
||||
* Build: `pnpm build`
|
||||
* Lint: `pnpm lint`
|
||||
* Fix lints: `pnpm lint-fix`
|
||||
* Test: `pnpm test`
|
||||
* Run single test file: `pnpm test __test__/arrow.test.ts`
|
||||
|
||||
+15
-13
@@ -12,20 +12,22 @@ Typescript.
|
||||
* `src/`: Rust bindings source code
|
||||
* `lancedb/`: Typescript package source code
|
||||
* `__test__/`: Unit tests
|
||||
* `examples/`: An npm package with the examples shown in the documentation
|
||||
* `examples/`: A pnpm package with the examples shown in the documentation
|
||||
|
||||
## Development environment
|
||||
|
||||
To set up your development environment, you will need to install the following:
|
||||
|
||||
1. Node.js 14 or later
|
||||
2. Rust's package manager, Cargo. Use [rustup](https://rustup.rs/) to install.
|
||||
3. [protoc](https://grpc.io/docs/protoc-installation/) (Protocol Buffers compiler)
|
||||
1. Node.js 22 or later (required by pnpm 11)
|
||||
2. [pnpm](https://pnpm.io/installation) 11 or later (or run via `corepack enable`,
|
||||
which uses the `packageManager` field in `package.json`)
|
||||
3. Rust's package manager, Cargo. Use [rustup](https://rustup.rs/) to install.
|
||||
4. [protoc](https://grpc.io/docs/protoc-installation/) (Protocol Buffers compiler)
|
||||
|
||||
Initial setup:
|
||||
|
||||
```shell
|
||||
npm install
|
||||
pnpm install
|
||||
```
|
||||
|
||||
### Commit Hooks
|
||||
@@ -39,38 +41,38 @@ pre-commit install
|
||||
|
||||
## Development
|
||||
|
||||
Most common development commands can be run using the npm scripts.
|
||||
Most common development commands can be run using the pnpm scripts.
|
||||
|
||||
Build the package
|
||||
|
||||
```shell
|
||||
npm install
|
||||
npm run build
|
||||
pnpm install
|
||||
pnpm build
|
||||
```
|
||||
|
||||
Lint:
|
||||
|
||||
```shell
|
||||
npm run lint
|
||||
pnpm lint
|
||||
```
|
||||
|
||||
Format and fix lints:
|
||||
|
||||
```shell
|
||||
npm run lint-fix
|
||||
pnpm lint-fix
|
||||
```
|
||||
|
||||
Run tests:
|
||||
|
||||
```shell
|
||||
npm test
|
||||
pnpm test
|
||||
```
|
||||
|
||||
To run a single test:
|
||||
|
||||
```shell
|
||||
# Single file: table.test.ts
|
||||
npm test -- table.test.ts
|
||||
pnpm test -- table.test.ts
|
||||
# Single test: 'merge insert' in table.test.ts
|
||||
npm test -- table.test.ts --testNamePattern=merge\ insert
|
||||
pnpm test -- table.test.ts --testNamePattern=merge\ insert
|
||||
```
|
||||
|
||||
@@ -22,6 +22,7 @@ arrow-schema.workspace = true
|
||||
env_logger.workspace = true
|
||||
futures.workspace = true
|
||||
lancedb = { path = "../rust/lancedb", default-features = false }
|
||||
lance-namespace.workspace = true
|
||||
napi = { version = "3.8.3", default-features = false, features = [
|
||||
"napi9",
|
||||
"async"
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
import { readdirSync } from "fs";
|
||||
import { Field, Float64, Schema } from "apache-arrow";
|
||||
import * as tmp from "tmp";
|
||||
import { Connection, Table, connect } from "../lancedb";
|
||||
import { Connection, Table, connect, connectNamespace } from "../lancedb";
|
||||
import { LocalTable } from "../lancedb/table";
|
||||
|
||||
describe("when connecting", () => {
|
||||
@@ -306,3 +306,186 @@ describe("clone table functionality", () => {
|
||||
).rejects.toThrow("Deep clone is not yet implemented");
|
||||
});
|
||||
});
|
||||
|
||||
describe("namespaces", () => {
|
||||
let tmpDir: tmp.DirResult;
|
||||
let db: Connection;
|
||||
|
||||
beforeEach(async () => {
|
||||
tmpDir = tmp.dirSync({ unsafeCleanup: true });
|
||||
// The local DirectoryNamespace backend only supports child namespaces
|
||||
// when manifest mode is enabled (see lance-namespace-impls/src/dir.rs).
|
||||
db = await connect(tmpDir.name, {
|
||||
// biome-ignore lint/style/useNamingConvention: opaque backend property key, must match Rust
|
||||
namespaceClientProperties: { manifest_enabled: "true" },
|
||||
});
|
||||
});
|
||||
afterEach(() => tmpDir.removeCallback());
|
||||
|
||||
it("should create and describe a namespace", async () => {
|
||||
await db.createNamespace(["myns"]);
|
||||
const desc = await db.describeNamespace(["myns"]);
|
||||
expect(desc).toBeDefined();
|
||||
});
|
||||
|
||||
it("should list namespaces created at the root", async () => {
|
||||
await db.createNamespace(["alpha"]);
|
||||
await db.createNamespace(["beta"]);
|
||||
const list = await db.listNamespaces();
|
||||
expect(list.namespaces).toEqual(expect.arrayContaining(["alpha", "beta"]));
|
||||
});
|
||||
|
||||
it("should list child namespaces under a parent", async () => {
|
||||
await db.createNamespace(["parent"]);
|
||||
await db.createNamespace(["parent", "child"]);
|
||||
const list = await db.listNamespaces(["parent"]);
|
||||
expect(list.namespaces).toContain("child");
|
||||
});
|
||||
|
||||
it("should drop a namespace", async () => {
|
||||
await db.createNamespace(["ephemeral"]);
|
||||
await db.dropNamespace(["ephemeral"]);
|
||||
const list = await db.listNamespaces();
|
||||
expect(list.namespaces).not.toContain("ephemeral");
|
||||
});
|
||||
|
||||
it("should raise an error on any namespace op after close", async () => {
|
||||
await db.close();
|
||||
await expect(db.describeNamespace(["foo"])).rejects.toThrow(
|
||||
"Connection is closed",
|
||||
);
|
||||
await expect(db.listNamespaces()).rejects.toThrow("Connection is closed");
|
||||
await expect(db.createNamespace(["foo"])).rejects.toThrow(
|
||||
"Connection is closed",
|
||||
);
|
||||
await expect(db.dropNamespace(["foo"])).rejects.toThrow(
|
||||
"Connection is closed",
|
||||
);
|
||||
});
|
||||
|
||||
it("should raise an understandable error when describing a non-existent namespace", async () => {
|
||||
await expect(db.describeNamespace(["does-not-exist"])).rejects.toThrow(
|
||||
/not found/i,
|
||||
);
|
||||
});
|
||||
|
||||
it("should raise an error when creating a namespace that already exists", async () => {
|
||||
await db.createNamespace(["dup"]);
|
||||
await expect(db.createNamespace(["dup"])).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("should reject an unrecognized createNamespace mode with a clear error", async () => {
|
||||
await expect(
|
||||
// biome-ignore lint/suspicious/noExplicitAny: deliberately bypass TS to test runtime validation
|
||||
db.createNamespace(["x"], { mode: "frobnicate" as any }),
|
||||
).rejects.toThrow(/Invalid mode 'frobnicate'/);
|
||||
});
|
||||
|
||||
it("should reject an unrecognized dropNamespace mode with a clear error", async () => {
|
||||
await db.createNamespace(["x"]);
|
||||
await expect(
|
||||
// biome-ignore lint/suspicious/noExplicitAny: deliberately bypass TS to test runtime validation
|
||||
db.dropNamespace(["x"], { mode: "frobnicate" as any }),
|
||||
).rejects.toThrow(/Invalid mode 'frobnicate'/);
|
||||
});
|
||||
|
||||
it("should reject an unrecognized dropNamespace behavior with a clear error", async () => {
|
||||
await db.createNamespace(["x"]);
|
||||
await expect(
|
||||
// biome-ignore lint/suspicious/noExplicitAny: deliberately bypass TS to test runtime validation
|
||||
db.dropNamespace(["x"], { behavior: "frobnicate" as any }),
|
||||
).rejects.toThrow(/Invalid behavior 'frobnicate'/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("connectNamespace", () => {
|
||||
let tmpDir: tmp.DirResult;
|
||||
beforeEach(() => {
|
||||
tmpDir = tmp.dirSync({ unsafeCleanup: true });
|
||||
});
|
||||
afterEach(() => tmpDir.removeCallback());
|
||||
|
||||
it("connects via the dir implementation and supports table ops", async () => {
|
||||
const db = await connectNamespace("dir", { root: tmpDir.name });
|
||||
await db.createTable("users", [{ id: 1 }, { id: 2 }]);
|
||||
await expect(db.tableNames()).resolves.toContain("users");
|
||||
});
|
||||
|
||||
it("throws a clear error when implName is empty", async () => {
|
||||
await expect(connectNamespace("", {})).rejects.toThrow(
|
||||
"implName must be a non-empty string",
|
||||
);
|
||||
});
|
||||
|
||||
it("throws when the namespace implementation is unknown", async () => {
|
||||
await expect(connectNamespace("not-a-real-impl", {})).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("passes storage options through to the namespace", async () => {
|
||||
const db = await connectNamespace(
|
||||
"dir",
|
||||
{ root: tmpDir.name },
|
||||
{ storageOptions: { newTableDataStorageVersion: "stable" } },
|
||||
);
|
||||
await db.createTable("plumbing", [{ id: 1 }]);
|
||||
await expect(db.tableNames()).resolves.toContain("plumbing");
|
||||
});
|
||||
|
||||
it("supports child namespaces when manifestEnabled is true on the dir config", async () => {
|
||||
const writer = await connectNamespace("dir", {
|
||||
root: tmpDir.name,
|
||||
manifestEnabled: true,
|
||||
});
|
||||
await writer.createNamespace(["analytics"]);
|
||||
await writer.createTable("orders", [{ id: 1 }, { id: 2 }], ["analytics"]);
|
||||
await writer.close();
|
||||
|
||||
const reader = await connectNamespace("dir", {
|
||||
root: tmpDir.name,
|
||||
manifestEnabled: true,
|
||||
});
|
||||
await expect(reader.tableNames(["analytics"])).resolves.toContain("orders");
|
||||
const orders = await reader.openTable("orders", ["analytics"]);
|
||||
await expect(orders.countRows()).resolves.toBe(2);
|
||||
});
|
||||
|
||||
it("merges extraProperties into the dir config and is overridden by typed fields", async () => {
|
||||
// Two observable assertions:
|
||||
// - Typed `root` overrides extraProperties.root: createTable would fail
|
||||
// under the bogus path if the override didn't happen.
|
||||
// - extraProperties.manifest_enabled="false" is honored end-to-end. Child
|
||||
// namespaces require manifest mode (default true), so explicitly
|
||||
// disabling it via extraProperties must make createNamespace reject. If
|
||||
// extraProperties pass-through were silently broken, the default would
|
||||
// let createNamespace succeed.
|
||||
const db = await connectNamespace("dir", {
|
||||
root: tmpDir.name,
|
||||
extraProperties: {
|
||||
root: "/should/be/overridden",
|
||||
// biome-ignore lint/style/useNamingConvention: backend property key
|
||||
manifest_enabled: "false",
|
||||
},
|
||||
});
|
||||
await db.createTable("base", [{ id: 1 }]);
|
||||
await expect(db.tableNames()).resolves.toContain("base");
|
||||
await expect(db.createNamespace(["analytics"])).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("flows unknown top-level keys through when implName is dynamic (no silent drop)", async () => {
|
||||
// Routes via the third overload because `impl` is `string`, not the
|
||||
// literal `"dir"`. The dispatcher still notices the runtime value is
|
||||
// "dir", but unknown keys like `manifest_enabled` must not be silently
|
||||
// dropped during the conversion.
|
||||
//
|
||||
// Asserting a *negative* outcome (manifest disabled -> createNamespace
|
||||
// rejects) is required for observability, since the backend default for
|
||||
// `manifest_enabled` is true.
|
||||
const impl: string = "dir";
|
||||
const db = await connectNamespace(impl, {
|
||||
root: tmpDir.name,
|
||||
// biome-ignore lint/style/useNamingConvention: backend property key
|
||||
manifest_enabled: "false",
|
||||
});
|
||||
await expect(db.createNamespace(["mixed"])).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -115,6 +115,12 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
await expect(table.countRows()).resolves.toBe(1);
|
||||
});
|
||||
|
||||
it("should accept skipAutoCleanup on add()", async () => {
|
||||
await table.add([{ id: 1 }], { skipAutoCleanup: true });
|
||||
await table.add([{ id: 2 }], { skipAutoCleanup: true });
|
||||
await expect(table.countRows()).resolves.toBe(2);
|
||||
});
|
||||
|
||||
it("should let me close the table", async () => {
|
||||
expect(table.isOpen()).toBe(true);
|
||||
table.close();
|
||||
|
||||
Generated
-4810
File diff suppressed because it is too large
Load Diff
@@ -11,16 +11,17 @@
|
||||
"test": "node --experimental-vm-modules node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose",
|
||||
"lint": "biome check *.ts && biome format *.ts",
|
||||
"lint-ci": "biome ci .",
|
||||
"lint-fix": "biome check --write *.ts && npm run format",
|
||||
"lint-fix": "biome check --write *.ts && pnpm format",
|
||||
"format": "biome format --write *.ts"
|
||||
},
|
||||
"author": "Lance Devs",
|
||||
"license": "Apache-2.0",
|
||||
"packageManager": "pnpm@11.1.1",
|
||||
"dependencies": {
|
||||
"@huggingface/transformers": "^3.0.2",
|
||||
"@huggingface/transformers": "3.0.2",
|
||||
"@lancedb/lancedb": "file:../dist",
|
||||
"openai": "^4.29.2",
|
||||
"sharp": "^0.33.5"
|
||||
"openai": "4.29.2",
|
||||
"sharp": "0.33.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "^1.7.3",
|
||||
|
||||
Generated
+3466
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,13 @@
|
||||
# Block resolution of versions less than 24h old (Shai-Hulud window).
|
||||
# This is the pnpm 11 default but pinned here so it's visible to
|
||||
# reviewers and survives a future pnpm major flipping the default.
|
||||
minimumReleaseAge: 1440
|
||||
|
||||
# Fail install if a transitive dep tries to run an unapproved script.
|
||||
strictDepBuilds: true
|
||||
|
||||
allowBuilds:
|
||||
'@biomejs/biome': true
|
||||
onnxruntime-node: true
|
||||
protobufjs: true
|
||||
sharp: true
|
||||
@@ -16,6 +16,18 @@ import {
|
||||
} from "./arrow";
|
||||
import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
|
||||
import { Connection as LanceDbConnection } from "./native";
|
||||
import type {
|
||||
CreateNamespaceResponse,
|
||||
DescribeNamespaceResponse,
|
||||
DropNamespaceResponse,
|
||||
ListNamespacesResponse,
|
||||
} from "./native";
|
||||
export type {
|
||||
CreateNamespaceResponse,
|
||||
DescribeNamespaceResponse,
|
||||
DropNamespaceResponse,
|
||||
ListNamespacesResponse,
|
||||
};
|
||||
import { sanitizeTable } from "./sanitize";
|
||||
import { LocalTable, Table } from "./table";
|
||||
|
||||
@@ -110,6 +122,28 @@ export interface TableNamesOptions {
|
||||
/** An optional limit to the number of results to return. */
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
export interface ListNamespacesOptions {
|
||||
/** Token from a previous response for pagination. */
|
||||
pageToken?: string;
|
||||
/** An optional limit to the number of results to return. */
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
export interface CreateNamespaceOptions {
|
||||
/** Creation mode. */
|
||||
mode?: "create" | "exist_ok" | "overwrite";
|
||||
/** Properties to set on the new namespace. */
|
||||
properties?: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface DropNamespaceOptions {
|
||||
/** Whether to skip if the namespace doesn't exist, or fail. */
|
||||
mode?: "skip" | "fail";
|
||||
/** Refuse to drop if non-empty (restrict) or drop recursively (cascade). */
|
||||
behavior?: "restrict" | "cascade";
|
||||
}
|
||||
|
||||
/**
|
||||
* A LanceDB Connection that allows you to open tables and create new ones.
|
||||
*
|
||||
@@ -268,6 +302,69 @@ export abstract class Connection {
|
||||
*/
|
||||
abstract dropAllTables(namespacePath?: string[]): Promise<void>;
|
||||
|
||||
/**
|
||||
* Describe a namespace, returning its properties.
|
||||
*
|
||||
* @param {string[]} namespacePath - The namespace path to describe, in
|
||||
* parent → child order, e.g. `["analytics", "sales"]`.
|
||||
* @returns {Promise<DescribeNamespaceResponse>} The namespace's properties
|
||||
* (may be undefined if the namespace has none).
|
||||
*/
|
||||
abstract describeNamespace(
|
||||
namespacePath: string[],
|
||||
): Promise<DescribeNamespaceResponse>;
|
||||
|
||||
/**
|
||||
* List the immediate child namespaces under the given parent.
|
||||
*
|
||||
* Results may be paginated. To retrieve subsequent pages, pass the
|
||||
* `pageToken` returned by a previous call.
|
||||
*
|
||||
* @param {string[]} namespacePath - The parent namespace path. Defaults
|
||||
* to the root namespace if omitted.
|
||||
* @param {Partial<ListNamespacesOptions>} options - Pagination options
|
||||
* (`pageToken`, `limit`).
|
||||
* @returns {Promise<ListNamespacesResponse>} Child namespace names and
|
||||
* an optional token for fetching the next page.
|
||||
*/
|
||||
abstract listNamespaces(
|
||||
namespacePath?: string[],
|
||||
options?: Partial<ListNamespacesOptions>,
|
||||
): Promise<ListNamespacesResponse>;
|
||||
|
||||
/**
|
||||
* Create a new namespace at the given path.
|
||||
*
|
||||
* @param {string[]} namespacePath - The namespace path to create.
|
||||
* @param {Partial<CreateNamespaceOptions>} options - Creation `mode`
|
||||
* ("create" | "exist_ok" | "overwrite") and optional `properties`
|
||||
* to attach to the namespace.
|
||||
* @returns {Promise<CreateNamespaceResponse>} The properties of the
|
||||
* created namespace and an optional transaction id.
|
||||
*/
|
||||
abstract createNamespace(
|
||||
namespacePath: string[],
|
||||
options?: Partial<CreateNamespaceOptions>,
|
||||
): Promise<CreateNamespaceResponse>;
|
||||
|
||||
/**
|
||||
* Drop a namespace.
|
||||
*
|
||||
* Use `behavior: "cascade"` to also drop everything contained in the
|
||||
* namespace (sub-namespaces and tables). The default `"restrict"`
|
||||
* behavior refuses to drop a non-empty namespace.
|
||||
*
|
||||
* @param {string[]} namespacePath - The namespace path to drop.
|
||||
* @param {Partial<DropNamespaceOptions>} options - `mode` ("skip" | "fail"
|
||||
* for missing-namespace handling) and `behavior` ("restrict" | "cascade").
|
||||
* @returns {Promise<DropNamespaceResponse>} Any properties returned by
|
||||
* the server and an optional transaction id.
|
||||
*/
|
||||
abstract dropNamespace(
|
||||
namespacePath: string[],
|
||||
options?: Partial<DropNamespaceOptions>,
|
||||
): Promise<DropNamespaceResponse>;
|
||||
|
||||
/**
|
||||
* Clone a table from a source table.
|
||||
*
|
||||
@@ -515,6 +612,45 @@ export class LocalConnection extends Connection {
|
||||
async dropAllTables(namespacePath?: string[]): Promise<void> {
|
||||
return this.inner.dropAllTables(namespacePath ?? []);
|
||||
}
|
||||
|
||||
describeNamespace(
|
||||
namespacePath: string[],
|
||||
): Promise<DescribeNamespaceResponse> {
|
||||
return this.inner.describeNamespace(namespacePath);
|
||||
}
|
||||
|
||||
listNamespaces(
|
||||
namespacePath?: string[],
|
||||
options?: Partial<ListNamespacesOptions>,
|
||||
): Promise<ListNamespacesResponse> {
|
||||
return this.inner.listNamespaces(
|
||||
namespacePath ?? [],
|
||||
options?.pageToken,
|
||||
options?.limit,
|
||||
);
|
||||
}
|
||||
|
||||
createNamespace(
|
||||
namespacePath: string[],
|
||||
options?: Partial<CreateNamespaceOptions>,
|
||||
): Promise<CreateNamespaceResponse> {
|
||||
return this.inner.createNamespace(
|
||||
namespacePath,
|
||||
options?.mode,
|
||||
options?.properties,
|
||||
);
|
||||
}
|
||||
|
||||
dropNamespace(
|
||||
namespacePath: string[],
|
||||
options?: Partial<DropNamespaceOptions>,
|
||||
): Promise<DropNamespaceResponse> {
|
||||
return this.inner.dropNamespace(
|
||||
namespacePath,
|
||||
options?.mode,
|
||||
options?.behavior,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
} from "./connection";
|
||||
|
||||
import {
|
||||
ConnectNamespaceOptions,
|
||||
ConnectionOptions,
|
||||
Connection as LanceDbConnection,
|
||||
JsHeaderProvider as NativeJsHeaderProvider,
|
||||
@@ -22,6 +23,7 @@ export { JsHeaderProvider as NativeJsHeaderProvider } from "./native.js";
|
||||
export {
|
||||
AddColumnsSql,
|
||||
ConnectionOptions,
|
||||
ConnectNamespaceOptions,
|
||||
IndexStatistics,
|
||||
IndexConfig,
|
||||
ClientConfig,
|
||||
@@ -62,6 +64,13 @@ export {
|
||||
CreateTableOptions,
|
||||
TableNamesOptions,
|
||||
OpenTableOptions,
|
||||
ListNamespacesOptions,
|
||||
CreateNamespaceOptions,
|
||||
DropNamespaceOptions,
|
||||
ListNamespacesResponse,
|
||||
CreateNamespaceResponse,
|
||||
DropNamespaceResponse,
|
||||
DescribeNamespaceResponse,
|
||||
} from "./connection";
|
||||
|
||||
export { Session } from "./native.js";
|
||||
@@ -293,3 +302,197 @@ export async function connect(
|
||||
);
|
||||
return new LocalConnection(nativeConn);
|
||||
}
|
||||
|
||||
/**
|
||||
* Configuration for the built-in directory namespace (`"dir"`).
|
||||
*
|
||||
* The directory namespace stores tables under a single root path (local
|
||||
* filesystem or object storage URI). See
|
||||
* {@link https://docs.lancedb.com/namespaces} for the documented surface;
|
||||
* less-common knobs live under {@link DirNamespaceConfig.extraProperties}.
|
||||
*/
|
||||
export interface DirNamespaceConfig {
|
||||
/** Root path or URI containing the LanceDB tables. */
|
||||
root: string;
|
||||
/**
|
||||
* Whether to maintain a namespace manifest at the root. Required for
|
||||
* child namespaces. Defaults to true on the impl side.
|
||||
*/
|
||||
manifestEnabled?: boolean;
|
||||
/**
|
||||
* Additional raw properties passed verbatim to the namespace
|
||||
* implementation (e.g. `storage.*`, `credential_vendor.*`). Typed
|
||||
* fields above take precedence on key collision.
|
||||
*/
|
||||
extraProperties?: Record<string, string>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Configuration for the built-in REST namespace (`"rest"`).
|
||||
*
|
||||
* The REST namespace talks to a remote catalog server over HTTP. See
|
||||
* {@link https://docs.lancedb.com/namespaces} for the documented surface;
|
||||
* less-common knobs (TLS, metrics) live under
|
||||
* {@link RestNamespaceConfig.extraProperties}.
|
||||
*/
|
||||
export interface RestNamespaceConfig {
|
||||
/** Catalog endpoint URL. */
|
||||
uri: string;
|
||||
/**
|
||||
* HTTP headers forwarded with each request. Keys are passed through
|
||||
* as-is (e.g. `"x-api-key"`, `"Authorization"`).
|
||||
*/
|
||||
headers?: Record<string, string>;
|
||||
/**
|
||||
* Additional raw properties passed verbatim to the namespace
|
||||
* implementation (e.g. `tls.*`, `ops_metrics_enabled`, `delimiter`).
|
||||
* Typed fields above take precedence on key collision.
|
||||
*/
|
||||
extraProperties?: Record<string, string>;
|
||||
}
|
||||
|
||||
function dirConfigToProperties(
|
||||
config: DirNamespaceConfig,
|
||||
): Record<string, string> {
|
||||
// Spread the whole input so that unknown keys (e.g. a raw `manifest_enabled`
|
||||
// passed via the dynamic-impl path) flow through instead of being dropped.
|
||||
// Typed transformations layer on top.
|
||||
const { manifestEnabled, extraProperties, ...rest } = config;
|
||||
const properties: Record<string, string> = {
|
||||
...(extraProperties ?? {}),
|
||||
...(rest as Record<string, string>),
|
||||
};
|
||||
if (manifestEnabled !== undefined) {
|
||||
properties.manifest_enabled = String(manifestEnabled);
|
||||
}
|
||||
return properties;
|
||||
}
|
||||
|
||||
function restConfigToProperties(
|
||||
config: RestNamespaceConfig,
|
||||
): Record<string, string> {
|
||||
const { headers, extraProperties, ...rest } = config;
|
||||
const properties: Record<string, string> = {
|
||||
...(extraProperties ?? {}),
|
||||
...(rest as Record<string, string>),
|
||||
};
|
||||
if (headers) {
|
||||
for (const [name, value] of Object.entries(headers)) {
|
||||
properties[`headers.${name}`] = value;
|
||||
}
|
||||
}
|
||||
return properties;
|
||||
}
|
||||
|
||||
/**
|
||||
* Connect to a LanceDB database through a namespace.
|
||||
*
|
||||
* Unlike {@link connect}, which routes by URI scheme (local path vs.
|
||||
* `db://` cloud), `connectNamespace` always returns a namespace-backed
|
||||
* connection. The `implName` selects the namespace implementation:
|
||||
*
|
||||
* - `"dir"` — directory namespace, configured with {@link DirNamespaceConfig}.
|
||||
* - `"rest"` — remote REST catalog, configured with {@link RestNamespaceConfig}.
|
||||
* - Any other string — full module path for a custom implementation,
|
||||
* configured with a free-form string-keyed `properties` map.
|
||||
*
|
||||
* @example Typed dir namespace
|
||||
* ```ts
|
||||
* const db = await connectNamespace("dir", { root: "/path/to/db" });
|
||||
* await db.createTable("users", [{ id: 1 }]);
|
||||
* ```
|
||||
*
|
||||
* @example Typed REST namespace with auth headers
|
||||
* ```ts
|
||||
* const db = await connectNamespace("rest", {
|
||||
* uri: "https://catalog.example.com",
|
||||
* headers: { "x-api-key": process.env.CATALOG_KEY ?? "" },
|
||||
* });
|
||||
* ```
|
||||
*
|
||||
* @example Custom implementation with raw properties
|
||||
* ```ts
|
||||
* const db = await connectNamespace("my.custom.Namespace", {
|
||||
* endpoint: "...",
|
||||
* });
|
||||
* ```
|
||||
*/
|
||||
export function connectNamespace(
|
||||
implName: "dir",
|
||||
config: DirNamespaceConfig,
|
||||
options?: Partial<ConnectNamespaceOptions>,
|
||||
): Promise<Connection>;
|
||||
/**
|
||||
* Connect through the built-in REST namespace.
|
||||
*
|
||||
* Configured with {@link RestNamespaceConfig}. See the function-level
|
||||
* documentation above for the full surface, examples, and how this
|
||||
* relates to {@link connect}.
|
||||
*
|
||||
* @example
|
||||
* ```ts
|
||||
* const db = await connectNamespace("rest", {
|
||||
* uri: "https://catalog.example.com",
|
||||
* headers: { "x-api-key": process.env.CATALOG_KEY ?? "" },
|
||||
* });
|
||||
* ```
|
||||
*/
|
||||
export function connectNamespace(
|
||||
implName: "rest",
|
||||
config: RestNamespaceConfig,
|
||||
options?: Partial<ConnectNamespaceOptions>,
|
||||
): Promise<Connection>;
|
||||
/**
|
||||
* Connect through a custom namespace implementation by full module path,
|
||||
* configured with a free-form string-keyed `properties` map. Use the
|
||||
* typed overloads above for the built-in `"dir"` and `"rest"` impls.
|
||||
*
|
||||
* See the function-level documentation above for examples and how this
|
||||
* relates to {@link connect}.
|
||||
*
|
||||
* @example
|
||||
* ```ts
|
||||
* const db = await connectNamespace("my.custom.Namespace", {
|
||||
* endpoint: "...",
|
||||
* });
|
||||
* ```
|
||||
*/
|
||||
export function connectNamespace(
|
||||
implName: string,
|
||||
properties: Record<string, string>,
|
||||
options?: Partial<ConnectNamespaceOptions>,
|
||||
): Promise<Connection>;
|
||||
export async function connectNamespace(
|
||||
implName: string,
|
||||
configOrProperties:
|
||||
| DirNamespaceConfig
|
||||
| RestNamespaceConfig
|
||||
| Record<string, string>,
|
||||
options?: Partial<ConnectNamespaceOptions>,
|
||||
): Promise<Connection> {
|
||||
let properties: Record<string, string>;
|
||||
if (implName === "dir") {
|
||||
properties = dirConfigToProperties(
|
||||
configOrProperties as DirNamespaceConfig,
|
||||
);
|
||||
} else if (implName === "rest") {
|
||||
properties = restConfigToProperties(
|
||||
configOrProperties as RestNamespaceConfig,
|
||||
);
|
||||
} else {
|
||||
properties = configOrProperties as Record<string, string>;
|
||||
}
|
||||
|
||||
const finalOptions: ConnectNamespaceOptions = (options ??
|
||||
{}) as ConnectNamespaceOptions;
|
||||
finalOptions.storageOptions = cleanseStorageOptions(
|
||||
finalOptions.storageOptions,
|
||||
);
|
||||
|
||||
const nativeConn = await LanceDbConnection.newWithNamespace(
|
||||
implName,
|
||||
properties,
|
||||
finalOptions,
|
||||
);
|
||||
return new LocalConnection(nativeConn);
|
||||
}
|
||||
|
||||
@@ -87,6 +87,23 @@ export class MergeInsertBuilder {
|
||||
this.#schema,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Skip the automatic cleanup of old dataset versions that would otherwise
|
||||
* run as part of this merge insert's commit. Forwards to
|
||||
* `MergeInsertBuilder::skip_auto_cleanup` in lance-core.
|
||||
*
|
||||
* Useful for high-frequency writers that prefer to manage version cleanup
|
||||
* themselves, or writers without delete permissions on the underlying storage.
|
||||
*
|
||||
* @param skip - If true, the auto-cleanup step is skipped at commit time.
|
||||
*/
|
||||
skipAutoCleanup(skip: boolean): MergeInsertBuilder {
|
||||
return new MergeInsertBuilder(
|
||||
this.#native.skipAutoCleanup(skip),
|
||||
this.#schema,
|
||||
);
|
||||
}
|
||||
/**
|
||||
* Executes the merge insert operation
|
||||
*
|
||||
|
||||
+13
-1
@@ -56,6 +56,18 @@ export interface AddDataOptions {
|
||||
* If "overwrite" then the new data will replace the existing data in the table.
|
||||
*/
|
||||
mode: "append" | "overwrite";
|
||||
/**
|
||||
* If true, skip the automatic cleanup of old dataset versions that would
|
||||
* otherwise run as part of this write's commit. Forwards to
|
||||
* `WriteParams.skip_auto_cleanup` in lance-core.
|
||||
*
|
||||
* Useful for high-frequency writers that prefer to manage version cleanup
|
||||
* themselves (for example, via a separate periodic optimize job), or for
|
||||
* writers that don't have delete permissions on the underlying storage.
|
||||
*
|
||||
* Defaults to false.
|
||||
*/
|
||||
skipAutoCleanup?: boolean;
|
||||
}
|
||||
|
||||
export interface UpdateOptions {
|
||||
@@ -636,7 +648,7 @@ export class LocalTable extends Table {
|
||||
const schema = await this.schema();
|
||||
|
||||
const buffer = await fromDataToBuffer(data, undefined, schema);
|
||||
return await this.inner.add(buffer, mode);
|
||||
return await this.inner.add(buffer, mode, options?.skipAutoCleanup);
|
||||
}
|
||||
|
||||
async update(
|
||||
|
||||
Generated
-10452
File diff suppressed because it is too large
Load Diff
+15
-14
@@ -38,15 +38,15 @@
|
||||
"url": "https://github.com/lancedb/lancedb"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@aws-sdk/client-dynamodb": "^3.33.0",
|
||||
"@aws-sdk/client-kms": "^3.33.0",
|
||||
"@aws-sdk/client-s3": "^3.33.0",
|
||||
"@aws-sdk/client-dynamodb": "3.1003.0",
|
||||
"@aws-sdk/client-kms": "3.1003.0",
|
||||
"@aws-sdk/client-s3": "3.1003.0",
|
||||
"@biomejs/biome": "^1.7.3",
|
||||
"@jest/globals": "^29.7.0",
|
||||
"@napi-rs/cli": "^3.5.1",
|
||||
"@napi-rs/cli": "3.5.1",
|
||||
"@types/axios": "^0.14.0",
|
||||
"@types/jest": "^29.1.2",
|
||||
"@types/node": "^22.7.4",
|
||||
"@types/node": "22.7.4",
|
||||
"@types/tmp": "^0.2.6",
|
||||
"apache-arrow-15": "npm:apache-arrow@15.0.0",
|
||||
"apache-arrow-16": "npm:apache-arrow@16.0.0",
|
||||
@@ -57,9 +57,9 @@
|
||||
"shx": "^0.3.4",
|
||||
"tmp": "^0.2.3",
|
||||
"ts-jest": "^29.1.2",
|
||||
"typedoc": "^0.26.4",
|
||||
"typedoc-plugin-markdown": "^4.2.1",
|
||||
"typescript": "^5.5.4",
|
||||
"typedoc": "0.26.4",
|
||||
"typedoc-plugin-markdown": "4.2.1",
|
||||
"typescript": "5.5.4",
|
||||
"typescript-eslint": "^7.1.0"
|
||||
},
|
||||
"ava": {
|
||||
@@ -68,15 +68,16 @@
|
||||
"engines": {
|
||||
"node": ">= 18"
|
||||
},
|
||||
"packageManager": "pnpm@11.1.1",
|
||||
"cpu": ["x64", "arm64"],
|
||||
"os": ["darwin", "linux", "win32"],
|
||||
"scripts": {
|
||||
"artifacts": "napi artifacts",
|
||||
"build:debug": "napi build --platform --dts ../lancedb/native.d.ts --js ../lancedb/native.js --output-dir lancedb",
|
||||
"postbuild:debug": "shx mkdir -p dist && shx cp lancedb/*.node dist/",
|
||||
"postbuild:debug": "shx mkdir -p dist && shx cp lancedb/*.node dist/ && node -e \"require('fs').writeFileSync('dist/package.json', JSON.stringify({name:'@lancedb/lancedb',type:'commonjs'}))\"",
|
||||
"build:release": "napi build --platform --release --dts ../lancedb/native.d.ts --js ../lancedb/native.js --output-dir dist",
|
||||
"build": "npm run build:debug && npm run tsc",
|
||||
"build-release": "npm run build:release && npm run tsc",
|
||||
"build": "pnpm build:debug && pnpm tsc",
|
||||
"build-release": "pnpm build:release && pnpm tsc",
|
||||
"tsc": "tsc -b",
|
||||
"posttsc": "shx cp lancedb/native.d.ts dist/native.d.ts",
|
||||
"lint-ci": "biome ci .",
|
||||
@@ -86,7 +87,7 @@
|
||||
"lint-fix": "biome check --write . && biome format --write .",
|
||||
"prepublishOnly": "napi prepublish -t npm",
|
||||
"test": "jest --verbose",
|
||||
"integration": "S3_TEST=1 npm run test",
|
||||
"integration": "S3_TEST=1 pnpm test",
|
||||
"universal": "napi universalize",
|
||||
"version": "napi version"
|
||||
},
|
||||
@@ -94,8 +95,8 @@
|
||||
"reflect-metadata": "^0.2.2"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@huggingface/transformers": "^3.0.2",
|
||||
"openai": "^4.29.2"
|
||||
"@huggingface/transformers": "3.0.2",
|
||||
"openai": "4.29.2"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"apache-arrow": ">=15.0.0 <=18.1.0"
|
||||
|
||||
Generated
+7317
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,18 @@
|
||||
# Flat node_modules layout. The @napi-rs/cli build step fails to locate
|
||||
# the cdylib artifact under pnpm's isolated layout; the hoisted linker
|
||||
# mirrors npm's structure and unblocks the native build.
|
||||
nodeLinker: hoisted
|
||||
|
||||
# Block resolution of versions less than 24h old (Shai-Hulud window).
|
||||
# This is the pnpm 11 default but pinned here so it's visible to
|
||||
# reviewers and survives a future pnpm major flipping the default.
|
||||
minimumReleaseAge: 1440
|
||||
|
||||
# Fail install if a transitive dep tries to run an unapproved script.
|
||||
strictDepBuilds: true
|
||||
|
||||
allowBuilds:
|
||||
'@biomejs/biome': true
|
||||
onnxruntime-node: true
|
||||
protobufjs: true
|
||||
sharp: true
|
||||
+187
-1
@@ -8,12 +8,16 @@ use lancedb::database::{CreateTableMode, Database};
|
||||
use napi::bindgen_prelude::*;
|
||||
use napi_derive::*;
|
||||
|
||||
use crate::ConnectNamespaceOptions;
|
||||
use crate::ConnectionOptions;
|
||||
use crate::error::NapiErrorExt;
|
||||
use crate::header::JsHeaderProvider;
|
||||
use crate::table::Table;
|
||||
use lancedb::connection::{ConnectBuilder, Connection as LanceDBConnection};
|
||||
use lancedb::connection::{ConnectBuilder, Connection as LanceDBConnection, connect_namespace};
|
||||
|
||||
use lance_namespace::models::{
|
||||
CreateNamespaceRequest, DescribeNamespaceRequest, DropNamespaceRequest, ListNamespacesRequest,
|
||||
};
|
||||
use lancedb::ipc::{ipc_file_to_batches, ipc_file_to_schema};
|
||||
|
||||
#[napi]
|
||||
@@ -21,6 +25,29 @@ pub struct Connection {
|
||||
inner: Option<LanceDBConnection>,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
pub struct DescribeNamespaceResponse {
|
||||
pub properties: Option<HashMap<String, String>>,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
pub struct ListNamespacesResponse {
|
||||
pub namespaces: Vec<String>,
|
||||
pub page_token: Option<String>,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
pub struct CreateNamespaceResponse {
|
||||
pub properties: Option<HashMap<String, String>>,
|
||||
pub transaction_id: Option<String>,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
pub struct DropNamespaceResponse {
|
||||
pub properties: Option<HashMap<String, String>>,
|
||||
pub transaction_id: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
impl Connection {
|
||||
pub(crate) fn inner_new(inner: LanceDBConnection) -> Self {
|
||||
Self { inner: Some(inner) }
|
||||
@@ -106,6 +133,39 @@ impl Connection {
|
||||
Ok(Self::inner_new(builder.execute().await.default_error()?))
|
||||
}
|
||||
|
||||
/// Create a new Connection instance backed by a namespace implementation.
|
||||
#[napi(factory)]
|
||||
pub async fn new_with_namespace(
|
||||
impl_name: String,
|
||||
properties: HashMap<String, String>,
|
||||
options: ConnectNamespaceOptions,
|
||||
) -> napi::Result<Self> {
|
||||
if impl_name.is_empty() {
|
||||
return Err(napi::Error::from_reason(
|
||||
"implName must be a non-empty string",
|
||||
));
|
||||
}
|
||||
|
||||
let mut builder = connect_namespace(&impl_name, properties);
|
||||
if let Some(interval) = options.read_consistency_interval {
|
||||
builder =
|
||||
builder.read_consistency_interval(std::time::Duration::from_secs_f64(interval));
|
||||
}
|
||||
if let Some(storage_options) = options.storage_options {
|
||||
for (key, value) in storage_options {
|
||||
builder = builder.storage_option(key, value);
|
||||
}
|
||||
}
|
||||
if let Some(namespace_client_properties) = options.namespace_client_properties {
|
||||
builder = builder.namespace_client_properties(namespace_client_properties);
|
||||
}
|
||||
if let Some(session) = options.session {
|
||||
builder = builder.session(session.inner.clone());
|
||||
}
|
||||
|
||||
Ok(Self::inner_new(builder.execute().await.default_error()?))
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn display(&self) -> napi::Result<String> {
|
||||
Ok(self.get_inner()?.to_string())
|
||||
@@ -273,4 +333,130 @@ impl Connection {
|
||||
let ns = namespace_path.unwrap_or_default();
|
||||
self.get_inner()?.drop_all_tables(&ns).await.default_error()
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
/// Describe a namespace and return its properties.
|
||||
pub async fn describe_namespace(
|
||||
&self,
|
||||
namespace_path: Vec<String>,
|
||||
) -> napi::Result<DescribeNamespaceResponse> {
|
||||
let req = DescribeNamespaceRequest {
|
||||
id: Some(namespace_path),
|
||||
..Default::default()
|
||||
};
|
||||
let resp = self
|
||||
.get_inner()?
|
||||
.describe_namespace(req)
|
||||
.await
|
||||
.default_error()?;
|
||||
Ok(DescribeNamespaceResponse {
|
||||
properties: resp.properties,
|
||||
})
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
/// List child namespaces under the given namespace path
|
||||
pub async fn list_namespaces(
|
||||
&self,
|
||||
namespace_path: Option<Vec<String>>,
|
||||
page_token: Option<String>,
|
||||
limit: Option<u32>,
|
||||
) -> napi::Result<ListNamespacesResponse> {
|
||||
let req = ListNamespacesRequest {
|
||||
id: namespace_path,
|
||||
page_token,
|
||||
limit: limit.map(|l| l as i32),
|
||||
..Default::default()
|
||||
};
|
||||
let resp = self
|
||||
.get_inner()?
|
||||
.list_namespaces(req)
|
||||
.await
|
||||
.default_error()?;
|
||||
Ok(ListNamespacesResponse {
|
||||
namespaces: resp.namespaces,
|
||||
page_token: resp.page_token,
|
||||
})
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
/// Create a new namespace with optional properties.
|
||||
pub async fn create_namespace(
|
||||
&self,
|
||||
namespace_path: Vec<String>,
|
||||
mode: Option<String>,
|
||||
properties: Option<HashMap<String, String>>,
|
||||
) -> napi::Result<CreateNamespaceResponse> {
|
||||
let mode_str = mode
|
||||
.map(|m| match m.to_lowercase().as_str() {
|
||||
"create" => Ok("Create".to_string()),
|
||||
"exist_ok" => Ok("ExistOk".to_string()),
|
||||
"overwrite" => Ok("Overwrite".to_string()),
|
||||
_ => Err(napi::Error::from_reason(format!(
|
||||
"Invalid mode '{}': expected one of 'create', 'exist_ok', 'overwrite'",
|
||||
m
|
||||
))),
|
||||
})
|
||||
.transpose()?;
|
||||
let req = CreateNamespaceRequest {
|
||||
id: Some(namespace_path),
|
||||
mode: mode_str,
|
||||
properties,
|
||||
..Default::default()
|
||||
};
|
||||
let resp = self
|
||||
.get_inner()?
|
||||
.create_namespace(req)
|
||||
.await
|
||||
.default_error()?;
|
||||
Ok(CreateNamespaceResponse {
|
||||
properties: resp.properties,
|
||||
transaction_id: resp.transaction_id,
|
||||
})
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
/// Drop a namespace.
|
||||
pub async fn drop_namespace(
|
||||
&self,
|
||||
namespace_path: Vec<String>,
|
||||
mode: Option<String>,
|
||||
behavior: Option<String>,
|
||||
) -> napi::Result<DropNamespaceResponse> {
|
||||
let mode_str = mode
|
||||
.map(|m| match m.to_lowercase().as_str() {
|
||||
"skip" => Ok("Skip".to_string()),
|
||||
"fail" => Ok("Fail".to_string()),
|
||||
_ => Err(napi::Error::from_reason(format!(
|
||||
"Invalid mode '{}': expected one of 'skip', 'fail'",
|
||||
m
|
||||
))),
|
||||
})
|
||||
.transpose()?;
|
||||
let behavior_str = behavior
|
||||
.map(|b| match b.to_lowercase().as_str() {
|
||||
"restrict" => Ok("Restrict".to_string()),
|
||||
"cascade" => Ok("Cascade".to_string()),
|
||||
_ => Err(napi::Error::from_reason(format!(
|
||||
"Invalid behavior '{}': expected one of 'restrict', 'cascade'",
|
||||
b
|
||||
))),
|
||||
})
|
||||
.transpose()?;
|
||||
let req = DropNamespaceRequest {
|
||||
id: Some(namespace_path),
|
||||
mode: mode_str,
|
||||
behavior: behavior_str,
|
||||
..Default::default()
|
||||
};
|
||||
let resp = self
|
||||
.get_inner()?
|
||||
.drop_namespace(req)
|
||||
.await
|
||||
.default_error()?;
|
||||
Ok(DropNamespaceResponse {
|
||||
properties: resp.properties,
|
||||
transaction_id: resp.transaction_id,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,6 +67,26 @@ pub struct OpenTableOptions {
|
||||
pub storage_options: Option<HashMap<String, String>>,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
#[derive(Debug)]
|
||||
pub struct ConnectNamespaceOptions {
|
||||
/// The interval, in seconds, at which to check for updates to the table
|
||||
/// from other processes. If None, then consistency is not checked. For
|
||||
/// performance reasons, this is the default. For strong consistency, set
|
||||
/// this to zero seconds. Then every read will check for updates from other
|
||||
/// processes. As a compromise, you can set this to a non-zero value for
|
||||
/// eventual consistency.
|
||||
pub read_consistency_interval: Option<f64>,
|
||||
/// Configuration for object storage. The available options are described
|
||||
/// at https://docs.lancedb.com/storage/
|
||||
pub storage_options: Option<HashMap<String, String>>,
|
||||
/// Extra properties for the backing namespace client.
|
||||
pub namespace_client_properties: Option<HashMap<String, String>>,
|
||||
/// The session to use for this connection. Holds shared caches and other
|
||||
/// session-specific state.
|
||||
pub session: Option<session::Session>,
|
||||
}
|
||||
|
||||
#[napi_derive::module_init]
|
||||
fn init() {
|
||||
let env = Env::new()
|
||||
|
||||
@@ -50,6 +50,13 @@ impl NativeMergeInsertBuilder {
|
||||
this
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn skip_auto_cleanup(&self, skip: bool) -> Self {
|
||||
let mut this = self.clone();
|
||||
this.inner.skip_auto_cleanup(skip);
|
||||
this
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn execute(&self, buf: Buffer) -> napi::Result<MergeResult> {
|
||||
let data = ipc_file_to_batches(buf.to_vec())
|
||||
|
||||
+14
-2
@@ -6,7 +6,7 @@ use std::collections::HashMap;
|
||||
use lancedb::ipc::{ipc_file_to_batches, ipc_file_to_schema};
|
||||
use lancedb::table::{
|
||||
AddDataMode, ColumnAlteration as LanceColumnAlteration, Duration, NewColumnTransform,
|
||||
OptimizeAction, OptimizeOptions, Table as LanceDbTable,
|
||||
OptimizeAction, OptimizeOptions, Table as LanceDbTable, WriteOptions,
|
||||
};
|
||||
use napi::bindgen_prelude::*;
|
||||
use napi_derive::napi;
|
||||
@@ -68,7 +68,12 @@ impl Table {
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn add(&self, buf: Buffer, mode: String) -> napi::Result<AddResult> {
|
||||
pub async fn add(
|
||||
&self,
|
||||
buf: Buffer,
|
||||
mode: String,
|
||||
skip_auto_cleanup: Option<bool>,
|
||||
) -> napi::Result<AddResult> {
|
||||
let batches = ipc_file_to_batches(buf.to_vec())
|
||||
.map_err(|e| napi::Error::from_reason(format!("Failed to read IPC file: {}", e)))?;
|
||||
let batches = batches
|
||||
@@ -92,6 +97,13 @@ impl Table {
|
||||
return Err(napi::Error::from_reason(format!("Invalid mode: {}", mode)));
|
||||
};
|
||||
|
||||
if skip_auto_cleanup.unwrap_or(false) {
|
||||
op = op.write_options(WriteOptions {
|
||||
skip_auto_cleanup: true,
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
|
||||
let res = op.execute().await.default_error()?;
|
||||
Ok(res.into())
|
||||
}
|
||||
|
||||
@@ -849,6 +849,10 @@ impl ListingDatabase {
|
||||
write_params.mode = WriteMode::Overwrite;
|
||||
}
|
||||
|
||||
if request.write_options.skip_auto_cleanup {
|
||||
write_params.skip_auto_cleanup = true;
|
||||
}
|
||||
|
||||
write_params.session = Some(self.session.clone());
|
||||
|
||||
write_params
|
||||
@@ -2034,6 +2038,7 @@ mod tests {
|
||||
}),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let table = db
|
||||
@@ -2107,6 +2112,7 @@ mod tests {
|
||||
}),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let table = db
|
||||
|
||||
@@ -414,6 +414,10 @@ impl Database for LanceNamespaceDatabase {
|
||||
params.mode = WriteMode::Overwrite;
|
||||
}
|
||||
|
||||
if request.write_options.skip_auto_cleanup {
|
||||
params.skip_auto_cleanup = true;
|
||||
}
|
||||
|
||||
// Set up storage options if provided
|
||||
if let Some(storage_opts) = initial_storage_options {
|
||||
let store_params = params
|
||||
|
||||
@@ -234,6 +234,7 @@ mod test {
|
||||
.create_table("test", data)
|
||||
.write_options(WriteOptions {
|
||||
lance_write_params: Some(param),
|
||||
..Default::default()
|
||||
})
|
||||
.execute()
|
||||
.await;
|
||||
|
||||
@@ -189,6 +189,18 @@ pub struct WriteOptions {
|
||||
// Coming soon: https://github.com/lancedb/lancedb/issues/992
|
||||
// /// What behavior to take if the data contains invalid vectors
|
||||
// pub on_bad_vectors: BadVectorHandling,
|
||||
/// If true, skip the automatic cleanup of old dataset versions that would
|
||||
/// otherwise run during the commit. This forwards to
|
||||
/// [`WriteParams::skip_auto_cleanup`] in lance-core.
|
||||
///
|
||||
/// Useful for high-frequency writers that want to manage version cleanup
|
||||
/// themselves (e.g. via a periodic optimize job), or for writers that
|
||||
/// lack delete permissions on the underlying storage.
|
||||
///
|
||||
/// If `lance_write_params` is also set with `skip_auto_cleanup = true`,
|
||||
/// the cleanup is skipped. Setting this field to `true` forces the flag
|
||||
/// on regardless of `lance_write_params`.
|
||||
pub skip_auto_cleanup: bool,
|
||||
/// Advanced parameters that can be used to customize table creation
|
||||
///
|
||||
/// Overlapping `OpenTableBuilder` options (e.g. [AddDataBuilder::mode]) will take
|
||||
@@ -2283,7 +2295,8 @@ impl BaseTable for NativeTable {
|
||||
|
||||
let output = add.into_plan(&table_schema, &table_def)?;
|
||||
|
||||
let lance_params = output
|
||||
let skip_auto_cleanup = output.write_options.skip_auto_cleanup;
|
||||
let mut lance_params = output
|
||||
.write_options
|
||||
.lance_write_params
|
||||
.unwrap_or(WriteParams {
|
||||
@@ -2293,6 +2306,9 @@ impl BaseTable for NativeTable {
|
||||
},
|
||||
..Default::default()
|
||||
});
|
||||
if skip_auto_cleanup {
|
||||
lance_params.skip_auto_cleanup = true;
|
||||
}
|
||||
|
||||
// Repartition for write parallelism if beneficial.
|
||||
let plan = if num_partitions > 1 {
|
||||
|
||||
@@ -441,6 +441,7 @@ mod tests {
|
||||
.add(new_batch.clone())
|
||||
.write_options(WriteOptions {
|
||||
lance_write_params: Some(param),
|
||||
..Default::default()
|
||||
})
|
||||
.mode(AddDataMode::Append)
|
||||
.execute()
|
||||
@@ -761,4 +762,56 @@ mod tests {
|
||||
table2.add(struct_batch).execute().await.unwrap();
|
||||
assert_eq!(table2.count_rows(None).await.unwrap(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_add_skip_auto_cleanup() {
|
||||
// Verifies WriteOptions::skip_auto_cleanup is forwarded to lance-core's
|
||||
// WriteParams and actually suppresses the cleanup hook on commit.
|
||||
let tmp_dir = tempfile::tempdir().unwrap();
|
||||
let uri = tmp_dir.path().to_str().unwrap();
|
||||
let conn = connect(uri).execute().await.unwrap();
|
||||
|
||||
let batch = record_batch!(("id", Int64, [1, 2, 3])).unwrap();
|
||||
let table = conn.create_table("t", batch).execute().await.unwrap();
|
||||
// Cleanup on every commit, with `older_than = 0s` so prior versions are
|
||||
// immediately eligible.
|
||||
table
|
||||
.as_native()
|
||||
.unwrap()
|
||||
.update_config(vec![
|
||||
("lance.auto_cleanup.interval".to_string(), "1".to_string()),
|
||||
(
|
||||
"lance.auto_cleanup.older_than".to_string(),
|
||||
"0s".to_string(),
|
||||
),
|
||||
])
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Write several versions with skip_auto_cleanup; none should be removed.
|
||||
for i in 0..3 {
|
||||
let new_batch = record_batch!(("id", Int64, [10 + i])).unwrap();
|
||||
table
|
||||
.add(new_batch)
|
||||
.write_options(WriteOptions {
|
||||
skip_auto_cleanup: true,
|
||||
..Default::default()
|
||||
})
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
let versions_before = table.list_versions().await.unwrap().len();
|
||||
|
||||
// Now write one more without the flag; cleanup should run and prune.
|
||||
let new_batch = record_batch!(("id", Int64, [42])).unwrap();
|
||||
table.add(new_batch).execute().await.unwrap();
|
||||
let versions_after = table.list_versions().await.unwrap().len();
|
||||
|
||||
assert!(
|
||||
versions_after < versions_before,
|
||||
"auto-cleanup should have removed old versions once the skip flag was off \
|
||||
(before={versions_before}, after={versions_after})"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -219,6 +219,7 @@ impl ExecutionPlan for InsertExec {
|
||||
&& let Some(merged_txn) = merge_transactions(transactions)
|
||||
{
|
||||
let new_dataset = CommitBuilder::new(dataset.clone())
|
||||
.with_skip_auto_cleanup(write_params.skip_auto_cleanup)
|
||||
.execute(merged_txn)
|
||||
.await?;
|
||||
ds_wrapper.update(new_dataset);
|
||||
|
||||
@@ -528,6 +528,7 @@ mod tests {
|
||||
}),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
})
|
||||
.execute()
|
||||
.await
|
||||
@@ -589,6 +590,7 @@ mod tests {
|
||||
}),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
})
|
||||
.execute()
|
||||
.await
|
||||
|
||||
@@ -55,6 +55,7 @@ pub struct MergeInsertBuilder {
|
||||
pub(crate) when_not_matched_by_source_delete_filt: Option<String>,
|
||||
pub(crate) timeout: Option<Duration>,
|
||||
pub(crate) use_index: bool,
|
||||
pub(crate) skip_auto_cleanup: bool,
|
||||
}
|
||||
|
||||
impl MergeInsertBuilder {
|
||||
@@ -69,6 +70,7 @@ impl MergeInsertBuilder {
|
||||
when_not_matched_by_source_delete_filt: None,
|
||||
timeout: None,
|
||||
use_index: true,
|
||||
skip_auto_cleanup: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -148,6 +150,17 @@ impl MergeInsertBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
/// Skip the automatic cleanup of old dataset versions that would otherwise
|
||||
/// run during the merge insert commit.
|
||||
///
|
||||
/// This forwards to [`lance::dataset::MergeInsertBuilder::skip_auto_cleanup`]
|
||||
/// in lance-core. Useful for high-frequency writers that want to manage
|
||||
/// version cleanup themselves, or writers without delete permissions.
|
||||
pub fn skip_auto_cleanup(&mut self, skip: bool) -> &mut Self {
|
||||
self.skip_auto_cleanup = skip;
|
||||
self
|
||||
}
|
||||
|
||||
/// Executes the merge insert operation
|
||||
///
|
||||
/// Returns version and statistics about the merge operation including the number of rows
|
||||
@@ -191,6 +204,9 @@ pub(crate) async fn execute_merge_insert(
|
||||
builder.when_not_matched_by_source(WhenNotMatchedBySource::Keep);
|
||||
}
|
||||
builder.use_index(params.use_index);
|
||||
if params.skip_auto_cleanup {
|
||||
builder.skip_auto_cleanup(true);
|
||||
}
|
||||
|
||||
let future = if let Some(timeout) = params.timeout {
|
||||
let future = builder
|
||||
|
||||
Reference in New Issue
Block a user