mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-27 08:28:28 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 0eead75e05 |
@@ -1,20 +0,0 @@
|
||||
{
|
||||
"name": "lancedb",
|
||||
"interface": {
|
||||
"displayName": "LanceDB"
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
"name": "lancedb",
|
||||
"source": {
|
||||
"source": "local",
|
||||
"path": "./plugins/lancedb"
|
||||
},
|
||||
"policy": {
|
||||
"installation": "AVAILABLE",
|
||||
"authentication": "ON_INSTALL"
|
||||
},
|
||||
"category": "Developer Tools"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -5,7 +5,3 @@ This directory contains repo-scoped code agent skills for the LanceDB project.
|
||||
Each skill is a folder that contains a required `SKILL.md` and optional bundled resources.
|
||||
|
||||
Codex discovers skills from `.agents/skills` in the current working directory and parent directories.
|
||||
|
||||
The `lancedb` skill lives in the `plugins/lancedb` plugin (see `plugins/lancedb/skills/lancedb`)
|
||||
so it can be installed via the plugin marketplaces (`.claude-plugin/marketplace.json` and
|
||||
`.agents/plugins/marketplace.json`); the `lancedb` entry here is a symlink into that plugin.
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
../../plugins/lancedb/skills/lancedb
|
||||
+1
-3
@@ -1,5 +1,5 @@
|
||||
[tool.bumpversion]
|
||||
current_version = "0.33.0-beta.0"
|
||||
current_version = "0.30.1-beta.2"
|
||||
parse = """(?x)
|
||||
(?P<major>0|[1-9]\\d*)\\.
|
||||
(?P<minor>0|[1-9]\\d*)\\.
|
||||
@@ -23,8 +23,6 @@ allow_dirty = true
|
||||
commit = true
|
||||
message = "Bump version: {current_version} → {new_version}"
|
||||
commit_args = ""
|
||||
# bump-my-version >=1.4.0 rejects pre_commit_hooks containing shell syntax unless opted in.
|
||||
allow_shell_hooks = true
|
||||
|
||||
# Java maven files
|
||||
pre_commit_hooks = [
|
||||
|
||||
@@ -1,19 +0,0 @@
|
||||
{
|
||||
"name": "lancedb",
|
||||
"owner": {
|
||||
"name": "LanceDB"
|
||||
},
|
||||
"description": "LanceDB plugins for Claude Code.",
|
||||
"plugins": [
|
||||
{
|
||||
"name": "lancedb",
|
||||
"source": "./plugins/lancedb",
|
||||
"description": "Write, review, debug, and document LanceDB pipelines in Python and TypeScript that work across local LanceDB OSS tables and remote LanceDB Enterprise/Cloud tables.",
|
||||
"version": "0.1.0",
|
||||
"author": {
|
||||
"name": "LanceDB"
|
||||
},
|
||||
"category": "development"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -18,14 +18,6 @@ inputs:
|
||||
description: "The manylinux version to build for"
|
||||
required: false
|
||||
default: "2_17"
|
||||
package-name:
|
||||
description: "Override [project] name in python/pyproject.toml (e.g. 'lancedb-compat'). Default keeps 'lancedb'."
|
||||
required: false
|
||||
default: "lancedb"
|
||||
rustflags:
|
||||
description: "RUSTFLAGS for the build container, as a single whitespace-free token (e.g. '-Ctarget-cpu=x86-64-v2'). Empty leaves RUSTFLAGS unset, keeping the defaults from .cargo/config.toml."
|
||||
required: false
|
||||
default: ""
|
||||
runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
@@ -35,18 +27,6 @@ runs:
|
||||
ARM_BUILD: ${{ inputs.arm-build }}
|
||||
run: |
|
||||
echo "ARM BUILD: $ARM_BUILD"
|
||||
- name: Patch package name for variant build
|
||||
if: ${{ inputs.package-name != 'lancedb' }}
|
||||
shell: bash
|
||||
env:
|
||||
PACKAGE_NAME: ${{ inputs.package-name }}
|
||||
run: |
|
||||
# Swap the [project] name so this build produces e.g. lancedb-compat
|
||||
# wheels. The package still installs files under the lancedb/
|
||||
# namespace -- import lancedb still works after pip install.
|
||||
sed -i.bak 's/^name = "lancedb"$/name = "'"$PACKAGE_NAME"'"/' python/pyproject.toml
|
||||
rm -f python/pyproject.toml.bak
|
||||
grep '^name = ' python/pyproject.toml
|
||||
- name: Build x86_64 Manylinux wheel
|
||||
if: ${{ inputs.arm-build == 'false' }}
|
||||
uses: PyO3/maturin-action@v1
|
||||
@@ -54,16 +34,15 @@ runs:
|
||||
maturin-version: "1.12.4"
|
||||
command: build
|
||||
working-directory: python
|
||||
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}"
|
||||
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/'"
|
||||
target: x86_64-unknown-linux-gnu
|
||||
manylinux: ${{ inputs.manylinux }}
|
||||
args: ${{ inputs.args }}
|
||||
before-script-linux: |
|
||||
set -e
|
||||
curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-x86_64.zip -o /tmp/protoc.zip
|
||||
unzip /tmp/protoc.zip -d /usr/local
|
||||
rm /tmp/protoc.zip
|
||||
/usr/local/bin/protoc --version
|
||||
curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-$(uname -m).zip > /tmp/protoc.zip \
|
||||
&& unzip /tmp/protoc.zip -d /usr/local \
|
||||
&& rm /tmp/protoc.zip
|
||||
- name: Build Arm Manylinux Wheel
|
||||
if: ${{ inputs.arm-build == 'true' }}
|
||||
uses: PyO3/maturin-action@v1
|
||||
@@ -71,14 +50,13 @@ runs:
|
||||
maturin-version: "1.12.4"
|
||||
command: build
|
||||
working-directory: python
|
||||
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}"
|
||||
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/'"
|
||||
target: aarch64-unknown-linux-gnu
|
||||
manylinux: ${{ inputs.manylinux }}
|
||||
args: ${{ inputs.args }}
|
||||
before-script-linux: |
|
||||
set -e
|
||||
yum install -y clang
|
||||
curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-aarch_64.zip -o /tmp/protoc.zip
|
||||
unzip /tmp/protoc.zip -d /usr/local
|
||||
rm /tmp/protoc.zip
|
||||
/usr/local/bin/protoc --version
|
||||
yum install -y clang \
|
||||
&& curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-aarch_64.zip > /tmp/protoc.zip \
|
||||
&& unzip /tmp/protoc.zip -d /usr/local \
|
||||
&& rm /tmp/protoc.zip
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
# Only runs on tags that matches the make-release action
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v4
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
workspaces: rust
|
||||
@@ -47,7 +47,7 @@ jobs:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v4
|
||||
- uses: ./.github/actions/create-failure-issue
|
||||
with:
|
||||
job-results: ${{ toJSON(needs) }}
|
||||
|
||||
@@ -36,14 +36,14 @@ jobs:
|
||||
echo "guidelines = ${{ inputs.guidelines }}"
|
||||
|
||||
- name: Checkout Repo
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.branch }}
|
||||
fetch-depth: 0
|
||||
persist-credentials: true
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v6
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
# pnpm 11 (used by the nodejs install step below) requires
|
||||
# Node >= 22.13; use 24 since 22 hits EOL in October.
|
||||
@@ -82,7 +82,7 @@ jobs:
|
||||
cache: maven
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v6
|
||||
uses: pnpm/action-setup@v4
|
||||
with:
|
||||
version: 11.1.1
|
||||
- name: Install Node.js dependencies for TypeScript bindings
|
||||
|
||||
@@ -30,13 +30,13 @@ jobs:
|
||||
echo "tag = ${{ inputs.tag || 'latest' }}"
|
||||
|
||||
- name: Checkout Repo LanceDB
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: true
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v6
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 20
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
name: Verify PR title / description conforms to semantic-release
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/setup-node@v6
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "18"
|
||||
# These rules are disabled because Github will always ensure there
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v4
|
||||
- name: Install dependencies needed for ubuntu
|
||||
run: |
|
||||
sudo apt install -y protobuf-compiler libssl-dev
|
||||
@@ -53,7 +53,7 @@ jobs:
|
||||
python -m pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -e .
|
||||
python -m pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -r ../docs/requirements.txt
|
||||
- name: Set up node
|
||||
uses: actions/setup-node@v6
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 20
|
||||
cache: 'npm'
|
||||
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
working-directory: ./java
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v4
|
||||
- name: Set up Java 8
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
@@ -73,7 +73,7 @@ jobs:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v4
|
||||
- uses: ./.github/actions/create-failure-issue
|
||||
with:
|
||||
job-results: ${{ toJSON(needs) }}
|
||||
|
||||
@@ -36,7 +36,7 @@ jobs:
|
||||
working-directory: ./java
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v4
|
||||
- name: Set up Java 17
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
|
||||
@@ -19,7 +19,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v4
|
||||
- name: Install license-header-checker
|
||||
working-directory: /tmp
|
||||
run: |
|
||||
|
||||
@@ -49,7 +49,7 @@ jobs:
|
||||
steps:
|
||||
- name: Output Inputs
|
||||
run: echo "${{ toJSON(github.event.inputs) }}"
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
lfs: true
|
||||
@@ -87,7 +87,7 @@ jobs:
|
||||
bash ci/update_lockfiles.sh --amend
|
||||
- name: Push new version tag
|
||||
if: ${{ !inputs.dry_run }}
|
||||
uses: ad-m/github-push-action@881a6320fdb16eb5318c5054f31c218aec2b324c # v1.3.0
|
||||
uses: ad-m/github-push-action@master
|
||||
with:
|
||||
# Need to use PAT here too to trigger next workflow. See comment above.
|
||||
github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
|
||||
|
||||
@@ -38,14 +38,14 @@ jobs:
|
||||
CC: gcc-12
|
||||
CXX: g++-12
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
lfs: true
|
||||
- uses: pnpm/action-setup@v6
|
||||
- uses: pnpm/action-setup@v4
|
||||
with:
|
||||
version: 11.1.1
|
||||
- uses: actions/setup-node@v6
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
|
||||
# in October. The library itself still supports Node >= 18
|
||||
@@ -86,14 +86,14 @@ jobs:
|
||||
shell: bash
|
||||
working-directory: nodejs
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
lfs: true
|
||||
- uses: pnpm/action-setup@v6
|
||||
- uses: pnpm/action-setup@v4
|
||||
with:
|
||||
version: 11.1.1
|
||||
- uses: actions/setup-node@v6
|
||||
- uses: actions/setup-node@v4
|
||||
name: Setup Node.js 24 for build
|
||||
with:
|
||||
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
|
||||
@@ -130,7 +130,7 @@ jobs:
|
||||
echo "Run 'pnpm run docs', fix any warnings, and commit the changes."
|
||||
exit 1
|
||||
fi
|
||||
- uses: actions/setup-node@v6
|
||||
- uses: actions/setup-node@v4
|
||||
name: Setup Node.js ${{ matrix.node-version }} for test
|
||||
with:
|
||||
node-version: ${{ matrix.node-version }}
|
||||
@@ -166,14 +166,14 @@ jobs:
|
||||
shell: bash
|
||||
working-directory: nodejs
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
lfs: true
|
||||
- uses: pnpm/action-setup@v6
|
||||
- uses: pnpm/action-setup@v4
|
||||
with:
|
||||
version: 11.1.1
|
||||
- uses: actions/setup-node@v6
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
|
||||
# in October.
|
||||
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
lfs: true
|
||||
@@ -103,7 +103,7 @@ jobs:
|
||||
features: fp16kernels
|
||||
pre_build: brew install protobuf
|
||||
- target: x86_64-pc-windows-msvc
|
||||
host: windows-2025-8x-x64
|
||||
host: windows-latest
|
||||
features: ","
|
||||
pre_build: |-
|
||||
choco install --no-progress protoc ninja nasm
|
||||
@@ -111,21 +111,12 @@ jobs:
|
||||
# There is an issue where choco doesn't add nasm to the path
|
||||
export PATH="$PATH:/c/Program Files/NASM"
|
||||
nasm -v
|
||||
# Fat LTO of the cdylib is single-threaded and the peak-memory
|
||||
# step of the build, and had started hitting rustc-LLVM OOM on the
|
||||
# Windows runners. ThinLTO parallelizes it across the runner's
|
||||
# cores and keeps peak memory well under the limit.
|
||||
export CARGO_PROFILE_RELEASE_LTO=thin
|
||||
export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
|
||||
- target: aarch64-pc-windows-msvc
|
||||
host: windows-2025-8x-x64
|
||||
host: windows-latest
|
||||
features: ","
|
||||
pre_build: |-
|
||||
choco install --no-progress protoc
|
||||
rustup target add aarch64-pc-windows-msvc
|
||||
# See ThinLTO note on the x86_64-pc-windows-msvc target above.
|
||||
export CARGO_PROFILE_RELEASE_LTO=thin
|
||||
export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
|
||||
- target: x86_64-unknown-linux-gnu
|
||||
host: ubuntu-latest
|
||||
features: fp16kernels
|
||||
@@ -179,13 +170,13 @@ jobs:
|
||||
run:
|
||||
working-directory: nodejs
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v4
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v6
|
||||
uses: pnpm/action-setup@v4
|
||||
with:
|
||||
version: 11.1.1
|
||||
- name: Setup node
|
||||
uses: actions/setup-node@v6
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
|
||||
# in October.
|
||||
@@ -199,7 +190,7 @@ jobs:
|
||||
toolchain: stable
|
||||
targets: ${{ matrix.settings.target }}
|
||||
- name: Cache cargo
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry/index/
|
||||
@@ -253,7 +244,7 @@ jobs:
|
||||
if: ${{ !matrix.settings.docker }}
|
||||
shell: bash
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: lancedb-${{ matrix.settings.target }}
|
||||
path: nodejs/dist/*.node
|
||||
@@ -265,7 +256,7 @@ jobs:
|
||||
run: pnpm tsc
|
||||
- name: Upload Generic Artifacts
|
||||
if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }}
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: nodejs-dist
|
||||
path: |
|
||||
@@ -296,13 +287,13 @@ jobs:
|
||||
shell: bash
|
||||
working-directory: nodejs
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v4
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v6
|
||||
uses: pnpm/action-setup@v4
|
||||
with:
|
||||
version: 11.1.1
|
||||
- name: Setup Node.js 24 for install
|
||||
uses: actions/setup-node@v6
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
|
||||
# in October.
|
||||
@@ -312,18 +303,18 @@ jobs:
|
||||
- name: Install dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
- name: Setup Node.js ${{ matrix.node }} for test
|
||||
uses: actions/setup-node@v6
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: ${{ matrix.node }}
|
||||
- name: Download artifacts
|
||||
uses: actions/download-artifact@v8
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: lancedb-${{ matrix.settings.target }}
|
||||
path: nodejs/dist/
|
||||
# For testing purposes:
|
||||
# run-id: 13982782871
|
||||
# github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
|
||||
- uses: actions/download-artifact@v8
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: nodejs-dist
|
||||
path: nodejs/dist
|
||||
@@ -348,13 +339,13 @@ jobs:
|
||||
needs:
|
||||
- test-lancedb
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v4
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v6
|
||||
uses: pnpm/action-setup@v4
|
||||
with:
|
||||
version: 11.1.1
|
||||
- name: Setup node
|
||||
uses: actions/setup-node@v6
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
cache: pnpm
|
||||
@@ -362,14 +353,14 @@ jobs:
|
||||
registry-url: "https://registry.npmjs.org"
|
||||
- name: Install dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
- uses: actions/download-artifact@v8
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: nodejs-dist
|
||||
path: nodejs/dist
|
||||
# For testing purposes:
|
||||
# run-id: 13982782871
|
||||
# github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
|
||||
- uses: actions/download-artifact@v8
|
||||
- uses: actions/download-artifact@v4
|
||||
name: Download arch-specific binaries
|
||||
with:
|
||||
pattern: lancedb-*
|
||||
@@ -407,7 +398,7 @@ jobs:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v4
|
||||
- uses: ./.github/actions/create-failure-issue
|
||||
with:
|
||||
job-results: ${{ toJSON(needs) }}
|
||||
|
||||
@@ -22,7 +22,7 @@ permissions:
|
||||
|
||||
jobs:
|
||||
linux:
|
||||
name: Python ${{ matrix.config.package_name }} ${{ matrix.config.platform }} manylinux${{ matrix.config.manylinux }}
|
||||
name: Python ${{ matrix.config.platform }} manylinux${{ matrix.config.manylinux }}
|
||||
timeout-minutes: 60
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -31,28 +31,11 @@ jobs:
|
||||
manylinux: "2_28"
|
||||
extra_args: "--features fp16kernels"
|
||||
runner: ubuntu-22.04
|
||||
package_name: "lancedb"
|
||||
rustflags: ""
|
||||
# For successful fat LTO builds, we need a large runner to avoid OOM errors.
|
||||
- platform: aarch64
|
||||
manylinux: "2_28"
|
||||
extra_args: "--features fp16kernels"
|
||||
runner: ubuntu-2404-8x-arm64
|
||||
package_name: "lancedb"
|
||||
rustflags: ""
|
||||
# `lancedb-compat`: pre-Haswell-friendly variant for x86_64 hosts
|
||||
# without AVX2 (Sandy Bridge / Ivy Bridge / Westmere on Intel,
|
||||
# Bulldozer / Piledriver / Steamroller on AMD). Compiled at the
|
||||
# `x86-64-v2` baseline; runtime SIMD dispatch in lance-linalg
|
||||
# picks the appropriate tier (scalar / AVX / AVX+FMA / AVX2+FMA
|
||||
# / AVX-512) at load time. Same import as `lancedb` -- conflicts
|
||||
# at install time, so users pick one.
|
||||
- platform: x86_64
|
||||
manylinux: "2_28"
|
||||
extra_args: ""
|
||||
runner: ubuntu-22.04
|
||||
package_name: "lancedb-compat"
|
||||
rustflags: "-Ctarget-cpu=x86-64-v2"
|
||||
runs-on: ${{ matrix.config.runner }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
@@ -69,13 +52,11 @@ jobs:
|
||||
args: "--release --strip ${{ matrix.config.extra_args }}"
|
||||
arm-build: ${{ matrix.config.platform == 'aarch64' }}
|
||||
manylinux: ${{ matrix.config.manylinux }}
|
||||
package-name: ${{ matrix.config.package_name }}
|
||||
rustflags: ${{ matrix.config.rustflags }}
|
||||
- uses: actions/upload-artifact@v7
|
||||
if: startsWith(github.ref, 'refs/tags/python-v')
|
||||
with:
|
||||
name: wheels-linux-${{ matrix.config.package_name }}-${{ matrix.config.platform }}-${{ matrix.config.manylinux }}
|
||||
path: target/wheels/*.whl
|
||||
name: wheels-linux-${{ matrix.config.platform }}-${{ matrix.config.manylinux }}
|
||||
path: target/wheels/lancedb-*.whl
|
||||
if-no-files-found: error
|
||||
mac:
|
||||
timeout-minutes: 90
|
||||
@@ -164,7 +145,7 @@ jobs:
|
||||
FURY_TOKEN: ${{ secrets.FURY_TOKEN }}
|
||||
run: |
|
||||
shopt -s nullglob
|
||||
WHEELS=(target/wheels/*.whl)
|
||||
WHEELS=(target/wheels/lancedb-*.whl)
|
||||
if [[ ${#WHEELS[@]} -eq 0 ]]; then
|
||||
echo "No wheels found in target/wheels/" >&2
|
||||
exit 1
|
||||
|
||||
@@ -41,7 +41,7 @@ jobs:
|
||||
shell: bash
|
||||
working-directory: python
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
lfs: true
|
||||
@@ -66,7 +66,7 @@ jobs:
|
||||
shell: bash
|
||||
working-directory: python
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
lfs: true
|
||||
@@ -95,7 +95,7 @@ jobs:
|
||||
shell: bash
|
||||
working-directory: python
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
lfs: true
|
||||
@@ -126,7 +126,7 @@ jobs:
|
||||
shell: bash
|
||||
working-directory: python
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
lfs: true
|
||||
@@ -160,7 +160,7 @@ jobs:
|
||||
shell: bash
|
||||
working-directory: python
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
lfs: true
|
||||
@@ -189,7 +189,7 @@ jobs:
|
||||
shell: bash
|
||||
working-directory: python
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
lfs: true
|
||||
@@ -212,7 +212,7 @@ jobs:
|
||||
shell: bash
|
||||
working-directory: python
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
lfs: true
|
||||
|
||||
+13
-29
@@ -40,7 +40,7 @@ jobs:
|
||||
CC: clang-18
|
||||
CXX: clang++-18
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
lfs: true
|
||||
@@ -65,7 +65,7 @@ jobs:
|
||||
timeout-minutes: 10
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v4
|
||||
- uses: EmbarkStudios/cargo-deny-action@v2
|
||||
with:
|
||||
command: check advisories bans licenses sources
|
||||
@@ -78,7 +78,7 @@ jobs:
|
||||
CC: clang
|
||||
CXX: clang++
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v4
|
||||
# Building without a lock file often requires the latest Rust version since downstream
|
||||
# dependencies may have updated their minimum Rust version.
|
||||
- uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
@@ -98,7 +98,7 @@ jobs:
|
||||
cargo build --profile ci --benches --all-features --tests
|
||||
|
||||
linux:
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 30
|
||||
# To build all features, we need more disk space than is available
|
||||
# on the free OSS github runner. This is mostly due to the the
|
||||
# sentence-transformers feature.
|
||||
@@ -113,7 +113,7 @@ jobs:
|
||||
CXX: clang++-18
|
||||
GH_TOKEN: ${{ secrets.SOPHON_READ_TOKEN }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
lfs: true
|
||||
@@ -125,26 +125,10 @@ jobs:
|
||||
- uses: rui314/setup-mold@v1
|
||||
- name: Make Swap
|
||||
run: |
|
||||
swapfile=/swapfile
|
||||
min_swap_bytes=$((15 * 1024 * 1024 * 1024))
|
||||
active_swap_bytes="$(sudo swapon --show=NAME,SIZE --bytes --noheadings | awk '$1 == "/swapfile" { print $2 }')"
|
||||
if [ -n "$active_swap_bytes" ]; then
|
||||
if [ "$active_swap_bytes" -ge "$min_swap_bytes" ]; then
|
||||
echo "/swapfile is already active with enough space; skipping swap creation"
|
||||
exit 0
|
||||
fi
|
||||
echo "/swapfile is already active but smaller than 16G; using /mnt/lancedb-swapfile"
|
||||
swapfile=/mnt/lancedb-swapfile
|
||||
fi
|
||||
if sudo swapon --show=NAME --noheadings | grep -Fxq "$swapfile"; then
|
||||
echo "$swapfile is already active; skipping swap creation"
|
||||
exit 0
|
||||
fi
|
||||
sudo rm -f "$swapfile"
|
||||
sudo fallocate -l 16G "$swapfile"
|
||||
sudo chmod 600 "$swapfile"
|
||||
sudo mkswap "$swapfile"
|
||||
sudo swapon "$swapfile"
|
||||
sudo fallocate -l 16G /swapfile
|
||||
sudo chmod 600 /swapfile
|
||||
sudo mkswap /swapfile
|
||||
sudo swapon /swapfile
|
||||
- name: Build
|
||||
run: cargo build --profile ci --all-features --tests --locked --examples
|
||||
- name: Run feature tests
|
||||
@@ -158,7 +142,7 @@ jobs:
|
||||
run: CARGO_ARGS="--profile ci" make -C ./lancedb remote-tests
|
||||
|
||||
macos:
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
mac-runner: ["macos-14", "macos-15"]
|
||||
@@ -168,7 +152,7 @@ jobs:
|
||||
shell: bash
|
||||
working-directory: rust
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
lfs: true
|
||||
@@ -197,7 +181,7 @@ jobs:
|
||||
run:
|
||||
working-directory: rust/lancedb
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v4
|
||||
- name: Set target
|
||||
run: rustup target add ${{ matrix.target }}
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
@@ -226,7 +210,7 @@ jobs:
|
||||
CC: clang-18
|
||||
CXX: clang++-18
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: true
|
||||
- name: Install dependencies
|
||||
|
||||
@@ -11,7 +11,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: main
|
||||
persist-credentials: false
|
||||
|
||||
@@ -11,7 +11,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: main
|
||||
persist-credentials: false
|
||||
|
||||
@@ -27,7 +27,6 @@ python/dist
|
||||
*.so
|
||||
*.dylib
|
||||
*.dll
|
||||
*.pdb
|
||||
|
||||
## Javascript
|
||||
*.node
|
||||
|
||||
Generated
+417
-1054
File diff suppressed because it is too large
Load Diff
+24
-26
@@ -13,25 +13,24 @@ categories = ["database-implementations"]
|
||||
rust-version = "1.91.0"
|
||||
|
||||
[workspace.dependencies]
|
||||
lance = { "version" = "=9.0.0", default-features = false }
|
||||
lance-core = "=9.0.0"
|
||||
lance-datagen = "=9.0.0"
|
||||
lance-file = "=9.0.0"
|
||||
lance-io = { "version" = "=9.0.0", default-features = false }
|
||||
lance-index = "=9.0.0"
|
||||
lance-linalg = "=9.0.0"
|
||||
lance-namespace = "=9.0.0"
|
||||
lance-namespace-impls = { "version" = "=9.0.0", default-features = false }
|
||||
lance-table = "=9.0.0"
|
||||
lance-testing = "=9.0.0"
|
||||
lance-datafusion = "=9.0.0"
|
||||
lance-encoding = "=9.0.0"
|
||||
lance-arrow = "=9.0.0"
|
||||
lance = { "version" = "=7.2.0-beta.3", default-features = false, "tag" = "v7.2.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=7.2.0-beta.3", "tag" = "v7.2.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=7.2.0-beta.3", "tag" = "v7.2.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=7.2.0-beta.3", "tag" = "v7.2.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=7.2.0-beta.3", default-features = false, "tag" = "v7.2.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=7.2.0-beta.3", "tag" = "v7.2.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=7.2.0-beta.3", "tag" = "v7.2.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=7.2.0-beta.3", "tag" = "v7.2.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=7.2.0-beta.3", default-features = false, "tag" = "v7.2.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=7.2.0-beta.3", "tag" = "v7.2.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=7.2.0-beta.3", "tag" = "v7.2.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=7.2.0-beta.3", "tag" = "v7.2.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=7.2.0-beta.3", "tag" = "v7.2.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=7.2.0-beta.3", "tag" = "v7.2.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
|
||||
ahash = "0.8"
|
||||
# Note that this one does not include pyarrow
|
||||
arrow = { version = "58.0.0", optional = false }
|
||||
arrow-array = "58.0.0"
|
||||
arrow-buffer = "58.0.0"
|
||||
arrow-data = "58.0.0"
|
||||
arrow-ipc = "58.0.0"
|
||||
arrow-ord = "58.0.0"
|
||||
@@ -39,23 +38,21 @@ arrow-schema = "58.0.0"
|
||||
arrow-select = "58.0.0"
|
||||
arrow-cast = "58.0.0"
|
||||
async-trait = "0"
|
||||
datafusion = { version = "54.0.0", default-features = false }
|
||||
datafusion-catalog = "54.0.0"
|
||||
datafusion-common = { version = "54.0.0", default-features = false }
|
||||
datafusion-execution = "54.0.0"
|
||||
datafusion-expr = "54.0.0"
|
||||
datafusion-functions = "54.0.0"
|
||||
datafusion-physical-plan = "54.0.0"
|
||||
datafusion-physical-expr = "54.0.0"
|
||||
datafusion-sql = "54.0.0"
|
||||
datafusion = { version = "53.0.0", default-features = false }
|
||||
datafusion-catalog = "53.0.0"
|
||||
datafusion-common = { version = "53.0.0", default-features = false }
|
||||
datafusion-execution = "53.0.0"
|
||||
datafusion-expr = "53.0.0"
|
||||
datafusion-functions = "53.0.0"
|
||||
datafusion-physical-plan = "53.0.0"
|
||||
datafusion-physical-expr = "53.0.0"
|
||||
datafusion-sql = "53.0.0"
|
||||
env_logger = "0.11"
|
||||
half = { "version" = "2.7.1", default-features = false, features = [
|
||||
"num-traits",
|
||||
] }
|
||||
futures = "0"
|
||||
log = "0.4"
|
||||
metrics = "0.24"
|
||||
metrics-util = "0.19"
|
||||
moka = { version = "0.12", features = ["future"] }
|
||||
object_store = "0.13.2"
|
||||
pin-project = "1.0.7"
|
||||
@@ -64,6 +61,7 @@ snafu = "0.8"
|
||||
url = "2"
|
||||
num-traits = "0.2"
|
||||
regex = "1.10"
|
||||
lazy_static = "1"
|
||||
semver = "1.0.25"
|
||||
chrono = "0.4"
|
||||
|
||||
|
||||
@@ -51,6 +51,18 @@ ignore = [
|
||||
# https://rustsec.org/advisories/RUSTSEC-2024-0436
|
||||
{ id = "RUSTSEC-2024-0436", reason = "transitive via datafusion; awaiting ecosystem migration" },
|
||||
|
||||
# encoding: unmaintained. Reached through lindera-dictionary, which is
|
||||
# required by the native Lindera tokenizer path. Lindera has not migrated
|
||||
# off this crate yet.
|
||||
# https://rustsec.org/advisories/RUSTSEC-2021-0153
|
||||
{ id = "RUSTSEC-2021-0153", reason = "transitive via lindera-dictionary for native Lindera tokenizer" },
|
||||
|
||||
# fast-float: unsound and unmaintained. Reached only through polars-arrow
|
||||
# from the optional Polars integration; replacement requires a Polars
|
||||
# dependency upgrade.
|
||||
# https://rustsec.org/advisories/RUSTSEC-2024-0379
|
||||
{ id = "RUSTSEC-2024-0379", reason = "transitive via polars-arrow; waiting on Polars migration" },
|
||||
|
||||
# tantivy: segfault on malformed input due to missing bounds check.
|
||||
# Pulled in via lance for full-text search. We only feed tantivy
|
||||
# documents we construct ourselves, not attacker-controlled bytes.
|
||||
@@ -68,6 +80,18 @@ ignore = [
|
||||
# https://rustsec.org/advisories/RUSTSEC-2025-0119
|
||||
{ id = "RUSTSEC-2025-0119", reason = "transitive via hf-hub/indicatif; cosmetic formatting crate" },
|
||||
|
||||
# bincode: unmaintained. Reached through lindera and lindera-dictionary,
|
||||
# which are required by the native Lindera tokenizer path. Lindera has not
|
||||
# migrated to another serialization format yet.
|
||||
# https://rustsec.org/advisories/RUSTSEC-2025-0141
|
||||
{ id = "RUSTSEC-2025-0141", reason = "transitive via lindera/lindera-dictionary for native Lindera tokenizer" },
|
||||
|
||||
# lru: soundness issue in IterMut. Reached only through aws-sdk-s3 in
|
||||
# LanceDB's dev-dependency graph; LanceDB does not use that iterator
|
||||
# directly. Clearing this requires the AWS SDK chain to update lru.
|
||||
# https://rustsec.org/advisories/RUSTSEC-2026-0002
|
||||
{ id = "RUSTSEC-2026-0002", reason = "transitive via aws-sdk-s3 dev-dependency; waiting on AWS SDK lru upgrade" },
|
||||
|
||||
# rustls-webpki 0.101.7 (old major line): name-constraint checks for
|
||||
# URI / wildcard names. Pulled in only via the legacy rustls 0.21 chain
|
||||
# from aws-smithy-http-client. The 0.103 line we actively use is patched.
|
||||
@@ -84,23 +108,11 @@ ignore = [
|
||||
# https://rustsec.org/advisories/RUSTSEC-2026-0104
|
||||
{ id = "RUSTSEC-2026-0104", reason = "only affects rustls-webpki 0.101 from legacy aws-smithy/rustls 0.21 chain" },
|
||||
|
||||
# pyo3 advisories in the Python bindings; tracked pending a patched pyo3 release.
|
||||
# https://rustsec.org/advisories/RUSTSEC-2026-0176
|
||||
# https://rustsec.org/advisories/RUSTSEC-2026-0177
|
||||
{ id = "RUSTSEC-2026-0176", reason = "pyo3 in Python bindings; awaiting patched pyo3 release" },
|
||||
{ id = "RUSTSEC-2026-0177", reason = "pyo3 in Python bindings; awaiting patched pyo3 release" },
|
||||
|
||||
# quick-xml < 0.41.0: quadratic runtime on duplicate attribute names (DoS).
|
||||
# quick-xml < 0.41.0: unbounded namespace-declaration allocation in NsReader (DoS).
|
||||
# Pulled in transitively by inferno (dev-only flame-graph dep), lance-namespace-impls
|
||||
# (git dep from lance), and opendal/reqsign (cloud storage XML parsing). The XML
|
||||
# parsed by opendal/reqsign comes from trusted cloud-storage endpoints (S3, GCS,
|
||||
# Azure), not attacker-controlled input. Clearing requires upstream crates to migrate
|
||||
# to quick-xml >= 0.41.0.
|
||||
# https://rustsec.org/advisories/RUSTSEC-2026-0194
|
||||
# https://rustsec.org/advisories/RUSTSEC-2026-0195
|
||||
{ id = "RUSTSEC-2026-0194", reason = "transitive via inferno/lance/opendal; XML from trusted cloud endpoints, not attacker-controlled" },
|
||||
{ id = "RUSTSEC-2026-0195", reason = "transitive via inferno/lance/opendal; XML from trusted cloud endpoints, not attacker-controlled" },
|
||||
# rand 0.8.5: soundness issue only when ThreadRng reseeds inside a custom
|
||||
# logger. Reached through several transitive chains. LanceDB does not use
|
||||
# rand from a custom logger; upgrade once all pinned chains accept 0.8.6+.
|
||||
# https://rustsec.org/advisories/RUSTSEC-2026-0097
|
||||
{ id = "RUSTSEC-2026-0097", reason = "transitive rand 0.8.5; LanceDB does not call ThreadRng from custom logging" },
|
||||
]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -135,14 +147,6 @@ allow = [
|
||||
"CDLA-Permissive-2.0",
|
||||
]
|
||||
confidence-threshold = 0.8
|
||||
# Per-crate license exceptions: allow a license for a specific crate only,
|
||||
# rather than globally via the `allow` list above.
|
||||
exceptions = [
|
||||
# CDDL-1.0 (copyleft) is pulled in only as a dev/profiling dependency via
|
||||
# `inferno` -> `pprof` -> `lance-testing`; it is a test dependency that we
|
||||
# do not distribute, so scope the allowance to `inferno` alone.
|
||||
{ allow = ["CDDL-1.0"], crate = "inferno" },
|
||||
]
|
||||
# Crates whose license cannot be determined from Cargo metadata but whose
|
||||
# license we've manually confirmed from upstream. Keep this list minimal.
|
||||
[[licenses.clarify]]
|
||||
|
||||
+30
-165
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
|
||||
<dependency>
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-core</artifactId>
|
||||
<version>0.33.0-beta.0</version>
|
||||
<version>0.30.1-beta.2</version>
|
||||
</dependency>
|
||||
```
|
||||
|
||||
@@ -249,57 +249,6 @@ try (BufferAllocator allocator = new RootAllocator();
|
||||
}
|
||||
```
|
||||
|
||||
### Creating an Empty Table
|
||||
|
||||
To create an empty table, send an Arrow IPC stream that contains the table schema and no record batches.
|
||||
The schema in the IPC stream becomes the table schema, and rows can be inserted later.
|
||||
|
||||
```java
|
||||
import org.lance.namespace.model.CreateTableRequest;
|
||||
import org.lance.namespace.model.CreateTableResponse;
|
||||
import org.apache.arrow.memory.BufferAllocator;
|
||||
import org.apache.arrow.memory.RootAllocator;
|
||||
import org.apache.arrow.vector.VectorSchemaRoot;
|
||||
import org.apache.arrow.vector.ipc.ArrowStreamWriter;
|
||||
import org.apache.arrow.vector.types.FloatingPointPrecision;
|
||||
import org.apache.arrow.vector.types.pojo.ArrowType;
|
||||
import org.apache.arrow.vector.types.pojo.Field;
|
||||
import org.apache.arrow.vector.types.pojo.FieldType;
|
||||
import org.apache.arrow.vector.types.pojo.Schema;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.nio.channels.Channels;
|
||||
import java.util.Arrays;
|
||||
|
||||
Schema schema = new Schema(Arrays.asList(
|
||||
new Field("id", FieldType.nullable(new ArrowType.Int(32, true)), null),
|
||||
new Field("name", FieldType.nullable(new ArrowType.Utf8()), null),
|
||||
new Field("embedding",
|
||||
FieldType.nullable(new ArrowType.FixedSizeList(128)),
|
||||
Arrays.asList(new Field("item",
|
||||
FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)),
|
||||
null)))
|
||||
));
|
||||
|
||||
byte[] emptyTableData;
|
||||
try (BufferAllocator allocator = new RootAllocator();
|
||||
VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) {
|
||||
root.setRowCount(0);
|
||||
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
try (ArrowStreamWriter writer = new ArrowStreamWriter(root, null, Channels.newChannel(out))) {
|
||||
writer.start();
|
||||
writer.end();
|
||||
}
|
||||
emptyTableData = out.toByteArray();
|
||||
}
|
||||
|
||||
CreateTableRequest request = new CreateTableRequest();
|
||||
request.setId(Arrays.asList("my_namespace", "empty_table"));
|
||||
|
||||
CreateTableResponse response = namespaceClient.createTable(request, emptyTableData);
|
||||
```
|
||||
|
||||
### Insert
|
||||
|
||||
```java
|
||||
@@ -482,88 +431,9 @@ query.setVector(vector);
|
||||
byte[] result = namespaceClient.queryTable(query);
|
||||
```
|
||||
|
||||
## Indexing
|
||||
### Reading Query Results
|
||||
|
||||
The Java SDK exposes the REST namespace index operations through the same `LanceNamespace` client.
|
||||
Index creation runs asynchronously, so use `listTableIndices` or `describeTableIndexStats` to check progress.
|
||||
|
||||
### Creating a Vector Index
|
||||
|
||||
```java
|
||||
import org.lance.namespace.model.CreateTableIndexRequest;
|
||||
import org.lance.namespace.model.CreateTableIndexResponse;
|
||||
|
||||
CreateTableIndexRequest request = new CreateTableIndexRequest();
|
||||
request.setId(Arrays.asList("my_namespace", "my_table"));
|
||||
request.setColumn("embedding");
|
||||
request.setIndexType("IVF_PQ");
|
||||
request.setDistanceType("cosine");
|
||||
request.setName("embedding_idx");
|
||||
|
||||
CreateTableIndexResponse response = namespaceClient.createTableIndex(request);
|
||||
System.out.println("Index transaction: " + response.getTransactionId());
|
||||
```
|
||||
|
||||
### Creating a Scalar Index
|
||||
|
||||
```java
|
||||
import org.lance.namespace.model.CreateTableIndexRequest;
|
||||
import org.lance.namespace.model.CreateTableScalarIndexResponse;
|
||||
|
||||
CreateTableIndexRequest request = new CreateTableIndexRequest();
|
||||
request.setId(Arrays.asList("my_namespace", "my_table"));
|
||||
request.setColumn("category");
|
||||
request.setIndexType("BTREE");
|
||||
request.setName("category_idx");
|
||||
|
||||
CreateTableScalarIndexResponse response = namespaceClient.createTableScalarIndex(request);
|
||||
System.out.println("Index transaction: " + response.getTransactionId());
|
||||
```
|
||||
|
||||
### Creating a Full Text Search Index
|
||||
|
||||
```java
|
||||
import org.lance.namespace.model.CreateTableIndexRequest;
|
||||
import org.lance.namespace.model.CreateTableScalarIndexResponse;
|
||||
|
||||
CreateTableIndexRequest request = new CreateTableIndexRequest();
|
||||
request.setId(Arrays.asList("my_namespace", "my_table"));
|
||||
request.setColumn("text_column");
|
||||
request.setIndexType("FTS");
|
||||
request.setName("text_idx");
|
||||
request.setBaseTokenizer("simple");
|
||||
request.setLowerCase(true);
|
||||
request.setWithPosition(true);
|
||||
|
||||
CreateTableScalarIndexResponse response = namespaceClient.createTableScalarIndex(request);
|
||||
System.out.println("Index transaction: " + response.getTransactionId());
|
||||
```
|
||||
|
||||
### Listing Indexes
|
||||
|
||||
```java
|
||||
import org.lance.namespace.model.IndexContent;
|
||||
import org.lance.namespace.model.ListTableIndicesRequest;
|
||||
import org.lance.namespace.model.ListTableIndicesResponse;
|
||||
|
||||
ListTableIndicesRequest request = new ListTableIndicesRequest();
|
||||
request.setId(Arrays.asList("my_namespace", "my_table"));
|
||||
|
||||
ListTableIndicesResponse response = namespaceClient.listTableIndices(request);
|
||||
for (IndexContent index : response.getIndexes()) {
|
||||
System.out.println(index.getIndexName() + ": " + index.getStatus());
|
||||
}
|
||||
```
|
||||
|
||||
!!! note
|
||||
The current Java namespace API exposes index type, index name, distance type, and full text search tokenizer options.
|
||||
IVF training parameters such as `num_partitions` are not exposed by `CreateTableIndexRequest` yet.
|
||||
To make those configurable from Java, the namespace API must add those fields first.
|
||||
|
||||
## Reading Query Results
|
||||
|
||||
Query results are returned as bytes in Apache Arrow IPC file format. Put the byte-channel
|
||||
adapter behind a small helper so query code can work with `ArrowFileReader` directly:
|
||||
Query results are returned in Apache Arrow IPC file format. Here's how to read them:
|
||||
|
||||
```java
|
||||
import org.apache.arrow.vector.ipc.ArrowFileReader;
|
||||
@@ -571,50 +441,45 @@ import org.apache.arrow.vector.VectorSchemaRoot;
|
||||
import org.apache.arrow.memory.BufferAllocator;
|
||||
import org.apache.arrow.memory.RootAllocator;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.channels.SeekableByteChannel;
|
||||
|
||||
final class ArrowIpc {
|
||||
static ArrowFileReader openFileReader(byte[] data, BufferAllocator allocator) throws IOException {
|
||||
return new ArrowFileReader(new ByteArraySeekableByteChannel(data), allocator);
|
||||
// Helper class to read Arrow data from byte array
|
||||
class ByteArraySeekableByteChannel implements SeekableByteChannel {
|
||||
private final byte[] data;
|
||||
private long position = 0;
|
||||
private boolean isOpen = true;
|
||||
|
||||
public ByteArraySeekableByteChannel(byte[] data) {
|
||||
this.data = data;
|
||||
}
|
||||
|
||||
private static final class ByteArraySeekableByteChannel implements SeekableByteChannel {
|
||||
private final byte[] data;
|
||||
private long position = 0;
|
||||
private boolean isOpen = true;
|
||||
|
||||
private ByteArraySeekableByteChannel(byte[] data) {
|
||||
this.data = data;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int read(ByteBuffer dst) {
|
||||
int remaining = dst.remaining();
|
||||
int available = (int) (data.length - position);
|
||||
if (available <= 0) return -1;
|
||||
int toRead = Math.min(remaining, available);
|
||||
dst.put(data, (int) position, toRead);
|
||||
position += toRead;
|
||||
return toRead;
|
||||
}
|
||||
|
||||
@Override public long position() { return position; }
|
||||
@Override public SeekableByteChannel position(long newPosition) { position = newPosition; return this; }
|
||||
@Override public long size() { return data.length; }
|
||||
@Override public boolean isOpen() { return isOpen; }
|
||||
@Override public void close() { isOpen = false; }
|
||||
@Override public int write(ByteBuffer src) { throw new UnsupportedOperationException(); }
|
||||
@Override public SeekableByteChannel truncate(long size) { throw new UnsupportedOperationException(); }
|
||||
@Override
|
||||
public int read(ByteBuffer dst) {
|
||||
int remaining = dst.remaining();
|
||||
int available = (int) (data.length - position);
|
||||
if (available <= 0) return -1;
|
||||
int toRead = Math.min(remaining, available);
|
||||
dst.put(data, (int) position, toRead);
|
||||
position += toRead;
|
||||
return toRead;
|
||||
}
|
||||
|
||||
@Override public long position() { return position; }
|
||||
@Override public SeekableByteChannel position(long newPosition) { position = newPosition; return this; }
|
||||
@Override public long size() { return data.length; }
|
||||
@Override public boolean isOpen() { return isOpen; }
|
||||
@Override public void close() { isOpen = false; }
|
||||
@Override public int write(ByteBuffer src) { throw new UnsupportedOperationException(); }
|
||||
@Override public SeekableByteChannel truncate(long size) { throw new UnsupportedOperationException(); }
|
||||
}
|
||||
|
||||
// Read query results
|
||||
byte[] queryResult = namespaceClient.queryTable(query);
|
||||
|
||||
try (BufferAllocator allocator = new RootAllocator();
|
||||
ArrowFileReader reader = ArrowIpc.openFileReader(queryResult, allocator)) {
|
||||
ArrowFileReader reader = new ArrowFileReader(
|
||||
new ByteArraySeekableByteChannel(queryResult), allocator)) {
|
||||
|
||||
for (int i = 0; i < reader.getRecordBlocks().size(); i++) {
|
||||
reader.loadRecordBatch(reader.getRecordBlocks().get(i));
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / BranchContents
|
||||
|
||||
# Class: BranchContents
|
||||
|
||||
## Constructors
|
||||
|
||||
### new BranchContents()
|
||||
|
||||
```ts
|
||||
new BranchContents(): BranchContents
|
||||
```
|
||||
|
||||
#### Returns
|
||||
|
||||
[`BranchContents`](BranchContents.md)
|
||||
|
||||
## Properties
|
||||
|
||||
### manifestSize
|
||||
|
||||
```ts
|
||||
manifestSize: number;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### parentBranch?
|
||||
|
||||
```ts
|
||||
optional parentBranch: string;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### parentVersion
|
||||
|
||||
```ts
|
||||
parentVersion: number;
|
||||
```
|
||||
@@ -1,139 +0,0 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / Branches
|
||||
|
||||
# Class: Branches
|
||||
|
||||
Branch manager for a [Table](Table.md).
|
||||
|
||||
Unlike tags, `create` and `checkout` return a new [Table](Table.md) handle scoped
|
||||
to the branch; writes on it do not affect `main`.
|
||||
|
||||
## Methods
|
||||
|
||||
### checkout()
|
||||
|
||||
```ts
|
||||
checkout(name, version?): Promise<Table>
|
||||
```
|
||||
|
||||
Check out an existing branch and return a handle scoped to it.
|
||||
|
||||
With `version` set, the returned handle is pinned to that version of the
|
||||
branch (a read-only, detached view); otherwise it tracks the branch's
|
||||
latest and stays writable.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **name**: `string`
|
||||
|
||||
* **version?**: `number`
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<[`Table`](Table.md)>
|
||||
|
||||
***
|
||||
|
||||
### create()
|
||||
|
||||
```ts
|
||||
create(
|
||||
name,
|
||||
fromRef?,
|
||||
fromVersion?): Promise<Table>
|
||||
```
|
||||
|
||||
Create a branch and return a handle scoped to it.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **name**: `string`
|
||||
Name of the new branch.
|
||||
|
||||
* **fromRef?**: `string`
|
||||
Source branch to fork from. Defaults to `main`.
|
||||
|
||||
* **fromVersion?**: `number`
|
||||
A specific version on `fromRef`. Defaults to latest.
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<[`Table`](Table.md)>
|
||||
|
||||
***
|
||||
|
||||
### delete()
|
||||
|
||||
```ts
|
||||
delete(name): Promise<void>
|
||||
```
|
||||
|
||||
Delete a branch.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **name**: `string`
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<`void`>
|
||||
|
||||
***
|
||||
|
||||
### diff()
|
||||
|
||||
```ts
|
||||
diff(fromBranch): Promise<BranchDiff>
|
||||
```
|
||||
|
||||
Compare a branch against main without modifying either branch.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **fromBranch**: `string`
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<[`BranchDiff`](../interfaces/BranchDiff.md)>
|
||||
|
||||
***
|
||||
|
||||
### list()
|
||||
|
||||
```ts
|
||||
list(): Promise<Record<string, BranchContents>>
|
||||
```
|
||||
|
||||
List all branches, mapping name to branch metadata.
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<`Record`<`string`, [`BranchContents`](BranchContents.md)>>
|
||||
|
||||
***
|
||||
|
||||
### merge()
|
||||
|
||||
```ts
|
||||
merge(fromBranch, dryRun): Promise<MergeBranchResult>
|
||||
```
|
||||
|
||||
Merge a branch into main.
|
||||
|
||||
Set `dryRun` to `true` to preview the merge. A rejected merge resolves
|
||||
with `status: "rejected"` instead of throwing.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **fromBranch**: `string`
|
||||
Branch to merge from.
|
||||
|
||||
* **dryRun**: `boolean` = `false`
|
||||
When true, only preview the merge. Defaults to false.
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<[`MergeBranchResult`](../interfaces/MergeBranchResult.md)>
|
||||
@@ -57,24 +57,6 @@ block size may be added in the future.
|
||||
|
||||
***
|
||||
|
||||
### fm()
|
||||
|
||||
```ts
|
||||
static fm(): Index
|
||||
```
|
||||
|
||||
Create an FM-Index.
|
||||
|
||||
An FM-Index is a scalar index on string or binary columns that accelerates
|
||||
substring search, i.e. `contains(col, 'needle')`. Unlike the tokenized
|
||||
full-text-search index, it matches arbitrary substrings of the raw bytes.
|
||||
|
||||
#### Returns
|
||||
|
||||
[`Index`](Index.md)
|
||||
|
||||
***
|
||||
|
||||
### fts()
|
||||
|
||||
```ts
|
||||
|
||||
@@ -33,7 +33,7 @@ protected inner: Query | Promise<Query>;
|
||||
### analyzePlan()
|
||||
|
||||
```ts
|
||||
analyzePlan(distributedMetrics?): Promise<string>
|
||||
analyzePlan(): Promise<string>
|
||||
```
|
||||
|
||||
Executes the query and returns the physical query plan annotated with runtime metrics.
|
||||
@@ -41,12 +41,6 @@ Executes the query and returns the physical query plan annotated with runtime me
|
||||
This is useful for debugging and performance analysis, as it shows how the query was executed
|
||||
and includes metrics such as elapsed time, rows processed, and I/O statistics.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **distributedMetrics?**: [`AnalyzePlanDistributedMetrics`](../type-aliases/AnalyzePlanDistributedMetrics.md)
|
||||
How distributed worker metrics are displayed for remote query plans.
|
||||
Defaults to `"aggregate"`.
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<`string`>
|
||||
@@ -524,9 +518,6 @@ x > 5 OR y = 'test'
|
||||
|
||||
Filtering performance can often be improved by creating a scalar index
|
||||
on the filter column(s).
|
||||
|
||||
Calling this multiple times combines the filters with a logical AND rather
|
||||
than replacing the previous filter.
|
||||
```
|
||||
|
||||
#### Inherited from
|
||||
|
||||
@@ -38,7 +38,7 @@ protected inner: NativeQueryType | Promise<NativeQueryType>;
|
||||
### analyzePlan()
|
||||
|
||||
```ts
|
||||
analyzePlan(distributedMetrics?): Promise<string>
|
||||
analyzePlan(): Promise<string>
|
||||
```
|
||||
|
||||
Executes the query and returns the physical query plan annotated with runtime metrics.
|
||||
@@ -46,12 +46,6 @@ Executes the query and returns the physical query plan annotated with runtime me
|
||||
This is useful for debugging and performance analysis, as it shows how the query was executed
|
||||
and includes metrics such as elapsed time, rows processed, and I/O statistics.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **distributedMetrics?**: [`AnalyzePlanDistributedMetrics`](../type-aliases/AnalyzePlanDistributedMetrics.md)
|
||||
How distributed worker metrics are displayed for remote query plans.
|
||||
Defaults to `"aggregate"`.
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<`string`>
|
||||
|
||||
@@ -110,23 +110,6 @@ containing the new version number of the table after altering the columns.
|
||||
|
||||
***
|
||||
|
||||
### branches()
|
||||
|
||||
```ts
|
||||
abstract branches(): Promise<Branches>
|
||||
```
|
||||
|
||||
Get the branch manager for this table.
|
||||
|
||||
Branches are isolated, writable lines of history forked from another
|
||||
branch (or version). Writes on a branch do not affect `main`.
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<[`Branches`](Branches.md)>
|
||||
|
||||
***
|
||||
|
||||
### checkout()
|
||||
|
||||
```ts
|
||||
@@ -295,23 +278,6 @@ await table.createIndex("my_float_col");
|
||||
|
||||
***
|
||||
|
||||
### currentBranch()
|
||||
|
||||
```ts
|
||||
abstract currentBranch(): null | string
|
||||
```
|
||||
|
||||
The branch this table handle is scoped to, or `null` for the main branch.
|
||||
|
||||
A handle returned by [Branches.create](Branches.md#create) or [Branches.checkout](Branches.md#checkout)
|
||||
reports the branch it targets; a handle opened normally reports `null`.
|
||||
|
||||
#### Returns
|
||||
|
||||
`null` \| `string`
|
||||
|
||||
***
|
||||
|
||||
### delete()
|
||||
|
||||
```ts
|
||||
@@ -398,26 +364,6 @@ Drop an index from the table.
|
||||
|
||||
***
|
||||
|
||||
### getLsmWriteSpec()
|
||||
|
||||
```ts
|
||||
abstract getLsmWriteSpec(): Promise<undefined | LsmWriteSpec>
|
||||
```
|
||||
|
||||
Read the [LsmWriteSpec](../interfaces/LsmWriteSpec.md) currently installed on this table.
|
||||
|
||||
Resolves to `undefined` when the MemWAL LSM write path is not enabled (no
|
||||
spec has been set, or it was removed with [Table#unsetLsmWriteSpec](Table.md#unsetlsmwritespec)).
|
||||
The returned spec — including its `maintainedIndexes` and
|
||||
`writerConfigDefaults` — mirrors what was passed to
|
||||
[Table#setLsmWriteSpec](Table.md#setlsmwritespec).
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<`undefined` \| [`LsmWriteSpec`](../interfaces/LsmWriteSpec.md)>
|
||||
|
||||
***
|
||||
|
||||
### indexStats()
|
||||
|
||||
```ts
|
||||
@@ -934,32 +880,6 @@ Return the table as an arrow table
|
||||
|
||||
***
|
||||
|
||||
### tokenize()
|
||||
|
||||
```ts
|
||||
abstract tokenize(query, options): Promise<FtsToken[]>
|
||||
```
|
||||
|
||||
Tokenize a full-text search query using the tokenizer configured on an FTS index.
|
||||
|
||||
Specify exactly one of `column` or `indexName`.
|
||||
|
||||
Model-backed tokenizers such as `jieba/*` and `lindera/*` are rebuilt in
|
||||
the client process from index metadata. For remote tables, this means the
|
||||
same tokenizer model files must also exist locally.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **query**: `string`
|
||||
|
||||
* **options**: [`TokenizeTableOptions`](../type-aliases/TokenizeTableOptions.md)
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<[`FtsToken`](../interfaces/FtsToken.md)[]>
|
||||
|
||||
***
|
||||
|
||||
### unsetLsmWriteSpec()
|
||||
|
||||
```ts
|
||||
|
||||
@@ -29,7 +29,7 @@ protected inner: TakeQuery | Promise<TakeQuery>;
|
||||
### analyzePlan()
|
||||
|
||||
```ts
|
||||
analyzePlan(distributedMetrics?): Promise<string>
|
||||
analyzePlan(): Promise<string>
|
||||
```
|
||||
|
||||
Executes the query and returns the physical query plan annotated with runtime metrics.
|
||||
@@ -37,12 +37,6 @@ Executes the query and returns the physical query plan annotated with runtime me
|
||||
This is useful for debugging and performance analysis, as it shows how the query was executed
|
||||
and includes metrics such as elapsed time, rows processed, and I/O statistics.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **distributedMetrics?**: [`AnalyzePlanDistributedMetrics`](../type-aliases/AnalyzePlanDistributedMetrics.md)
|
||||
How distributed worker metrics are displayed for remote query plans.
|
||||
Defaults to `"aggregate"`.
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<`string`>
|
||||
|
||||
@@ -51,7 +51,7 @@ addQueryVector(vector): VectorQuery
|
||||
### analyzePlan()
|
||||
|
||||
```ts
|
||||
analyzePlan(distributedMetrics?): Promise<string>
|
||||
analyzePlan(): Promise<string>
|
||||
```
|
||||
|
||||
Executes the query and returns the physical query plan annotated with runtime metrics.
|
||||
@@ -59,12 +59,6 @@ Executes the query and returns the physical query plan annotated with runtime me
|
||||
This is useful for debugging and performance analysis, as it shows how the query was executed
|
||||
and includes metrics such as elapsed time, rows processed, and I/O statistics.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **distributedMetrics?**: [`AnalyzePlanDistributedMetrics`](../type-aliases/AnalyzePlanDistributedMetrics.md)
|
||||
How distributed worker metrics are displayed for remote query plans.
|
||||
Defaults to `"aggregate"`.
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<`string`>
|
||||
@@ -773,9 +767,6 @@ x > 5 OR y = 'test'
|
||||
|
||||
Filtering performance can often be improved by creating a scalar index
|
||||
on the filter column(s).
|
||||
|
||||
Calling this multiple times combines the filters with a logical AND rather
|
||||
than replacing the previous filter.
|
||||
```
|
||||
|
||||
#### Inherited from
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / OAuthFlowType
|
||||
|
||||
# Enumeration: OAuthFlowType
|
||||
|
||||
OAuth authentication flow types.
|
||||
|
||||
## Enumeration Members
|
||||
|
||||
### AzureManagedIdentity
|
||||
|
||||
```ts
|
||||
AzureManagedIdentity: "azure_managed_identity";
|
||||
```
|
||||
|
||||
Azure Managed Identity via IMDS.
|
||||
|
||||
***
|
||||
|
||||
### ClientCredentials
|
||||
|
||||
```ts
|
||||
ClientCredentials: "client_credentials";
|
||||
```
|
||||
|
||||
Client Credentials grant (service-to-service / M2M).
|
||||
@@ -1,42 +0,0 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / instrumentLanceDbMetrics
|
||||
|
||||
# Function: instrumentLanceDbMetrics()
|
||||
|
||||
```ts
|
||||
function instrumentLanceDbMetrics(meterProvider?): boolean
|
||||
```
|
||||
|
||||
Register LanceDB metrics as OpenTelemetry observable instruments.
|
||||
|
||||
Installs a process-global metrics recorder and creates one observable
|
||||
instrument per LanceDB metric (currently object store request counts, bytes,
|
||||
latency, errors, and throttles) on the given (or global) `MeterProvider`. The
|
||||
configured `MetricReader` then collects them on its own schedule.
|
||||
|
||||
Counters and gauges map directly to observable counters/gauges. Because
|
||||
OpenTelemetry has no asynchronous histogram instrument, each histogram is
|
||||
exported Prometheus-style as cumulative `le` bucket counts (`<name>_bucket`,
|
||||
with an `le` attribute) plus `<name>_count` and `<name>_sum`.
|
||||
|
||||
Requires `@opentelemetry/api` (a dependency) and, to actually export, an
|
||||
OpenTelemetry SDK such as `@opentelemetry/sdk-metrics`.
|
||||
|
||||
## Parameters
|
||||
|
||||
* **meterProvider?**: `MeterProvider`
|
||||
The provider to register instruments on. Defaults to the
|
||||
global provider from `@opentelemetry/api`.
|
||||
|
||||
## Returns
|
||||
|
||||
`boolean`
|
||||
|
||||
`true` if the recorder is installed and instruments are registered.
|
||||
`false` if a different `metrics` recorder is already installed in this
|
||||
process (only one global recorder is permitted), in which case a warning is
|
||||
emitted and no instruments are created. Calling this more than once is safe;
|
||||
instruments are created only on the first successful call.
|
||||
@@ -1,26 +0,0 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / tokenize
|
||||
|
||||
# Function: tokenize()
|
||||
|
||||
```ts
|
||||
function tokenize(query, options?): Promise<FtsToken[]>
|
||||
```
|
||||
|
||||
Tokenize a full-text search query using an explicit tokenizer.
|
||||
|
||||
This does not require a table or FTS index. The tokenizer options match
|
||||
[Index.fts](../classes/Index.md#fts).
|
||||
|
||||
## Parameters
|
||||
|
||||
* **query**: `string`
|
||||
|
||||
* **options?**: `Partial`<[`TokenizeOptions`](../interfaces/TokenizeOptions.md)>
|
||||
|
||||
## Returns
|
||||
|
||||
`Promise`<[`FtsToken`](../interfaces/FtsToken.md)[]>
|
||||
@@ -12,7 +12,6 @@
|
||||
## Enumerations
|
||||
|
||||
- [FullTextQueryType](enumerations/FullTextQueryType.md)
|
||||
- [OAuthFlowType](enumerations/OAuthFlowType.md)
|
||||
- [Occur](enumerations/Occur.md)
|
||||
- [Operator](enumerations/Operator.md)
|
||||
|
||||
@@ -20,8 +19,6 @@
|
||||
|
||||
- [BooleanQuery](classes/BooleanQuery.md)
|
||||
- [BoostQuery](classes/BoostQuery.md)
|
||||
- [BranchContents](classes/BranchContents.md)
|
||||
- [Branches](classes/Branches.md)
|
||||
- [Connection](classes/Connection.md)
|
||||
- [HeaderProvider](classes/HeaderProvider.md)
|
||||
- [Index](classes/Index.md)
|
||||
@@ -52,11 +49,6 @@
|
||||
- [AddDataOptions](interfaces/AddDataOptions.md)
|
||||
- [AddResult](interfaces/AddResult.md)
|
||||
- [AlterColumnsResult](interfaces/AlterColumnsResult.md)
|
||||
- [BranchColumnChange](interfaces/BranchColumnChange.md)
|
||||
- [BranchColumnSummary](interfaces/BranchColumnSummary.md)
|
||||
- [BranchDiff](interfaces/BranchDiff.md)
|
||||
- [BranchIndexSummary](interfaces/BranchIndexSummary.md)
|
||||
- [BranchRowCountSummary](interfaces/BranchRowCountSummary.md)
|
||||
- [ClientConfig](interfaces/ClientConfig.md)
|
||||
- [ColumnAlteration](interfaces/ColumnAlteration.md)
|
||||
- [ColumnOrdering](interfaces/ColumnOrdering.md)
|
||||
@@ -77,7 +69,6 @@
|
||||
- [FragmentStatistics](interfaces/FragmentStatistics.md)
|
||||
- [FragmentSummaryStats](interfaces/FragmentSummaryStats.md)
|
||||
- [FtsOptions](interfaces/FtsOptions.md)
|
||||
- [FtsToken](interfaces/FtsToken.md)
|
||||
- [FullTextQuery](interfaces/FullTextQuery.md)
|
||||
- [FullTextSearchOptions](interfaces/FullTextSearchOptions.md)
|
||||
- [HnswPqOptions](interfaces/HnswPqOptions.md)
|
||||
@@ -91,12 +82,7 @@
|
||||
- [ListNamespacesOptions](interfaces/ListNamespacesOptions.md)
|
||||
- [ListNamespacesResponse](interfaces/ListNamespacesResponse.md)
|
||||
- [LsmWriteSpec](interfaces/LsmWriteSpec.md)
|
||||
- [MergeBlocker](interfaces/MergeBlocker.md)
|
||||
- [MergeBranchResult](interfaces/MergeBranchResult.md)
|
||||
- [MergePreview](interfaces/MergePreview.md)
|
||||
- [MergeResult](interfaces/MergeResult.md)
|
||||
- [NativeOAuthConfig](interfaces/NativeOAuthConfig.md)
|
||||
- [OAuthConfig](interfaces/OAuthConfig.md)
|
||||
- [OpenTableOptions](interfaces/OpenTableOptions.md)
|
||||
- [OptimizeOptions](interfaces/OptimizeOptions.md)
|
||||
- [OptimizeStats](interfaces/OptimizeStats.md)
|
||||
@@ -116,7 +102,6 @@
|
||||
- [TimeoutConfig](interfaces/TimeoutConfig.md)
|
||||
- [TlsConfig](interfaces/TlsConfig.md)
|
||||
- [TokenResponse](interfaces/TokenResponse.md)
|
||||
- [TokenizeOptions](interfaces/TokenizeOptions.md)
|
||||
- [UpdateFieldMetadataResult](interfaces/UpdateFieldMetadataResult.md)
|
||||
- [UpdateOptions](interfaces/UpdateOptions.md)
|
||||
- [UpdateResult](interfaces/UpdateResult.md)
|
||||
@@ -126,8 +111,6 @@
|
||||
|
||||
## Type Aliases
|
||||
|
||||
- [AnalyzePlanDistributedMetrics](type-aliases/AnalyzePlanDistributedMetrics.md)
|
||||
- [BaseTokenizer](type-aliases/BaseTokenizer.md)
|
||||
- [Data](type-aliases/Data.md)
|
||||
- [DataLike](type-aliases/DataLike.md)
|
||||
- [FieldLike](type-aliases/FieldLike.md)
|
||||
@@ -137,15 +120,12 @@
|
||||
- [RecordBatchLike](type-aliases/RecordBatchLike.md)
|
||||
- [SchemaLike](type-aliases/SchemaLike.md)
|
||||
- [TableLike](type-aliases/TableLike.md)
|
||||
- [TokenizeTableOptions](type-aliases/TokenizeTableOptions.md)
|
||||
|
||||
## Functions
|
||||
|
||||
- [RecordBatchIterator](functions/RecordBatchIterator.md)
|
||||
- [connect](functions/connect.md)
|
||||
- [connectNamespace](functions/connectNamespace.md)
|
||||
- [instrumentLanceDbMetrics](functions/instrumentLanceDbMetrics.md)
|
||||
- [makeArrowTable](functions/makeArrowTable.md)
|
||||
- [packBits](functions/packBits.md)
|
||||
- [permutationBuilder](functions/permutationBuilder.md)
|
||||
- [tokenize](functions/tokenize.md)
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / BranchColumnChange
|
||||
|
||||
# Interface: BranchColumnChange
|
||||
|
||||
A column whose definition differs between main and the branch.
|
||||
|
||||
## Properties
|
||||
|
||||
### branch
|
||||
|
||||
```ts
|
||||
branch: BranchColumnSummary;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### main
|
||||
|
||||
```ts
|
||||
main: BranchColumnSummary;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### name
|
||||
|
||||
```ts
|
||||
name: string;
|
||||
```
|
||||
@@ -1,33 +0,0 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / BranchColumnSummary
|
||||
|
||||
# Interface: BranchColumnSummary
|
||||
|
||||
Summary of a column in a branch diff.
|
||||
|
||||
## Properties
|
||||
|
||||
### dataType
|
||||
|
||||
```ts
|
||||
dataType: string;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### name
|
||||
|
||||
```ts
|
||||
name: string;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### nullable
|
||||
|
||||
```ts
|
||||
nullable: boolean;
|
||||
```
|
||||
@@ -1,129 +0,0 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / BranchDiff
|
||||
|
||||
# Interface: BranchDiff
|
||||
|
||||
Read-only comparison of a branch against main.
|
||||
|
||||
## Properties
|
||||
|
||||
### addedColumns
|
||||
|
||||
```ts
|
||||
addedColumns: BranchColumnSummary[];
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### addedIndexes
|
||||
|
||||
```ts
|
||||
addedIndexes: BranchIndexSummary[];
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### baseMoved
|
||||
|
||||
```ts
|
||||
baseMoved: boolean;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### branchVersion
|
||||
|
||||
```ts
|
||||
branchVersion: number;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### changedColumns
|
||||
|
||||
```ts
|
||||
changedColumns: BranchColumnChange[];
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### fromBranch
|
||||
|
||||
```ts
|
||||
fromBranch: string;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### mainVersion
|
||||
|
||||
```ts
|
||||
mainVersion: number;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### mergeBlockers
|
||||
|
||||
```ts
|
||||
mergeBlockers: MergeBlocker[];
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### mergeable
|
||||
|
||||
```ts
|
||||
mergeable: boolean;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### parentVersion
|
||||
|
||||
```ts
|
||||
parentVersion: number;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### removedColumns
|
||||
|
||||
```ts
|
||||
removedColumns: BranchColumnSummary[];
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### removedIndexes
|
||||
|
||||
```ts
|
||||
removedIndexes: BranchIndexSummary[];
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### rowCountBranch
|
||||
|
||||
```ts
|
||||
rowCountBranch: number;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### rowCountMain
|
||||
|
||||
```ts
|
||||
rowCountMain: number;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### rowSummary
|
||||
|
||||
```ts
|
||||
rowSummary: BranchRowCountSummary;
|
||||
```
|
||||
@@ -1,41 +0,0 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / BranchIndexSummary
|
||||
|
||||
# Interface: BranchIndexSummary
|
||||
|
||||
Summary of an index in a branch diff.
|
||||
|
||||
## Properties
|
||||
|
||||
### columns
|
||||
|
||||
```ts
|
||||
columns: string[];
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### indexName
|
||||
|
||||
```ts
|
||||
indexName: string;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### indexType?
|
||||
|
||||
```ts
|
||||
optional indexType: string;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### status
|
||||
|
||||
```ts
|
||||
status: string;
|
||||
```
|
||||
@@ -1,57 +0,0 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / BranchRowCountSummary
|
||||
|
||||
# Interface: BranchRowCountSummary
|
||||
|
||||
Row-level comparison between main and the branch.
|
||||
|
||||
## Properties
|
||||
|
||||
### deltaAvailable
|
||||
|
||||
```ts
|
||||
deltaAvailable: boolean;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### inputsChanged
|
||||
|
||||
```ts
|
||||
inputsChanged: number;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### newOnBase
|
||||
|
||||
```ts
|
||||
newOnBase: number;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### newOnBranch
|
||||
|
||||
```ts
|
||||
newOnBranch: number;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### staleRecompute
|
||||
|
||||
```ts
|
||||
staleRecompute: number;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### unchanged
|
||||
|
||||
```ts
|
||||
unchanged: number;
|
||||
```
|
||||
@@ -64,19 +64,6 @@ client used by manifest-enabled native connections.
|
||||
|
||||
***
|
||||
|
||||
### oauthConfig?
|
||||
|
||||
```ts
|
||||
optional oauthConfig: NativeOAuthConfig;
|
||||
```
|
||||
|
||||
(For LanceDB cloud only): OAuth configuration for IdP-based
|
||||
authentication (e.g., Azure Entra ID). When set, token acquisition
|
||||
and refresh are handled entirely in Rust. TypeScript users should pass
|
||||
the public `OAuthConfig` type exported from `@lancedb/lancedb`.
|
||||
|
||||
***
|
||||
|
||||
### readConsistencyInterval?
|
||||
|
||||
```ts
|
||||
|
||||
@@ -23,7 +23,7 @@ whether to remove punctuation
|
||||
### baseTokenizer?
|
||||
|
||||
```ts
|
||||
optional baseTokenizer: BaseTokenizer;
|
||||
optional baseTokenizer: "raw" | "simple" | "whitespace" | "ngram";
|
||||
```
|
||||
|
||||
The tokenizer to use when building the index.
|
||||
@@ -37,10 +37,6 @@ The following tokenizers are available:
|
||||
|
||||
"raw" - Raw tokenizer. This tokenizer does not split the text into tokens and indexes the entire text as a single token.
|
||||
|
||||
"icu" - ICU dictionary-based word segmentation.
|
||||
|
||||
"icu/split" - ICU segmentation with simple-style delimiter splitting.
|
||||
|
||||
***
|
||||
|
||||
### language?
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / FtsToken
|
||||
|
||||
# Interface: FtsToken
|
||||
|
||||
Token produced by the tokenizer configured on a full-text search index.
|
||||
|
||||
## Properties
|
||||
|
||||
### position
|
||||
|
||||
```ts
|
||||
position: number;
|
||||
```
|
||||
|
||||
Token position used by full-text query matching.
|
||||
|
||||
***
|
||||
|
||||
### text
|
||||
|
||||
```ts
|
||||
text: string;
|
||||
```
|
||||
|
||||
Token text after tokenizer filters have been applied.
|
||||
@@ -23,31 +23,6 @@ be more columns to represent composite indices.
|
||||
|
||||
***
|
||||
|
||||
### createdAt?
|
||||
|
||||
```ts
|
||||
optional createdAt: Date;
|
||||
```
|
||||
|
||||
When the index was created.
|
||||
|
||||
`undefined` for remote tables or indices created before timestamps were tracked.
|
||||
|
||||
***
|
||||
|
||||
### indexDetails?
|
||||
|
||||
```ts
|
||||
optional indexDetails: any;
|
||||
```
|
||||
|
||||
Index-type-specific details parsed as a JavaScript object.
|
||||
|
||||
Falls back to a raw string if JSON parsing fails. `undefined` for
|
||||
remote tables or when details are unavailable.
|
||||
|
||||
***
|
||||
|
||||
### indexType
|
||||
|
||||
```ts
|
||||
@@ -58,30 +33,6 @@ The type of the index
|
||||
|
||||
***
|
||||
|
||||
### indexUuid?
|
||||
|
||||
```ts
|
||||
optional indexUuid: string;
|
||||
```
|
||||
|
||||
The UUID of the first segment of the index.
|
||||
|
||||
`undefined` for remote tables, which do not yet surface this.
|
||||
|
||||
***
|
||||
|
||||
### indexVersion?
|
||||
|
||||
```ts
|
||||
optional indexVersion: number;
|
||||
```
|
||||
|
||||
The on-disk index format version.
|
||||
|
||||
`undefined` for remote tables.
|
||||
|
||||
***
|
||||
|
||||
### name
|
||||
|
||||
```ts
|
||||
@@ -89,63 +40,3 @@ name: string;
|
||||
```
|
||||
|
||||
The name of the index
|
||||
|
||||
***
|
||||
|
||||
### numIndexedRows?
|
||||
|
||||
```ts
|
||||
optional numIndexedRows: number;
|
||||
```
|
||||
|
||||
The number of rows indexed, across all segments.
|
||||
|
||||
`undefined` for remote tables.
|
||||
|
||||
***
|
||||
|
||||
### numSegments?
|
||||
|
||||
```ts
|
||||
optional numSegments: number;
|
||||
```
|
||||
|
||||
The number of segments that make up the index.
|
||||
|
||||
`undefined` for remote tables.
|
||||
|
||||
***
|
||||
|
||||
### numUnindexedRows?
|
||||
|
||||
```ts
|
||||
optional numUnindexedRows: number;
|
||||
```
|
||||
|
||||
The number of rows not yet covered by this index.
|
||||
|
||||
`undefined` for remote tables.
|
||||
|
||||
***
|
||||
|
||||
### sizeBytes?
|
||||
|
||||
```ts
|
||||
optional sizeBytes: number;
|
||||
```
|
||||
|
||||
The total size in bytes of all index files across all segments.
|
||||
|
||||
`undefined` for remote tables or indices without size tracking.
|
||||
|
||||
***
|
||||
|
||||
### typeUrl?
|
||||
|
||||
```ts
|
||||
optional typeUrl: string;
|
||||
```
|
||||
|
||||
The protobuf type URL, a precise type identifier for the index.
|
||||
|
||||
`undefined` for remote tables.
|
||||
|
||||
@@ -30,6 +30,17 @@ The type of the index
|
||||
|
||||
***
|
||||
|
||||
### loss?
|
||||
|
||||
```ts
|
||||
optional loss: number;
|
||||
```
|
||||
|
||||
The KMeans loss value of the index,
|
||||
it is only present for vector indices.
|
||||
|
||||
***
|
||||
|
||||
### numIndexedRows
|
||||
|
||||
```ts
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / MergeBlocker
|
||||
|
||||
# Interface: MergeBlocker
|
||||
|
||||
A reason why a branch cannot currently be merged.
|
||||
|
||||
## Properties
|
||||
|
||||
### code
|
||||
|
||||
```ts
|
||||
code: string;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### message
|
||||
|
||||
```ts
|
||||
message: string;
|
||||
```
|
||||
@@ -1,46 +0,0 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / MergeBranchResult
|
||||
|
||||
# Interface: MergeBranchResult
|
||||
|
||||
Result of previewing or attempting a branch merge.
|
||||
|
||||
## Properties
|
||||
|
||||
### diff
|
||||
|
||||
```ts
|
||||
diff: BranchDiff;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### mainVersionAfter?
|
||||
|
||||
```ts
|
||||
optional mainVersionAfter: number;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### preview
|
||||
|
||||
```ts
|
||||
preview: MergePreview;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### status
|
||||
|
||||
```ts
|
||||
status:
|
||||
| "unknown"
|
||||
| "rejected"
|
||||
| "ready"
|
||||
| "notImplemented"
|
||||
| "merged";
|
||||
```
|
||||
@@ -1,17 +0,0 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / MergePreview
|
||||
|
||||
# Interface: MergePreview
|
||||
|
||||
Changes that would be, or were, promoted by a branch merge.
|
||||
|
||||
## Properties
|
||||
|
||||
### promotedColumns
|
||||
|
||||
```ts
|
||||
promotedColumns: string[];
|
||||
```
|
||||
@@ -1,88 +0,0 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / NativeOAuthConfig
|
||||
|
||||
# Interface: NativeOAuthConfig
|
||||
|
||||
OAuth configuration for LanceDB authentication.
|
||||
|
||||
This is the generated napi-rs binding shape. TypeScript users should prefer
|
||||
the public `OAuthConfig` type exported from `@lancedb/lancedb`.
|
||||
|
||||
All token acquisition and refresh is handled in the Rust layer.
|
||||
|
||||
## Properties
|
||||
|
||||
### clientId
|
||||
|
||||
```ts
|
||||
clientId: string;
|
||||
```
|
||||
|
||||
Application / Client ID.
|
||||
|
||||
***
|
||||
|
||||
### clientSecret?
|
||||
|
||||
```ts
|
||||
optional clientSecret: string;
|
||||
```
|
||||
|
||||
Client secret (required for client_credentials).
|
||||
|
||||
***
|
||||
|
||||
### flow?
|
||||
|
||||
```ts
|
||||
optional flow: string;
|
||||
```
|
||||
|
||||
Authentication flow: "client_credentials" or "azure_managed_identity"
|
||||
|
||||
***
|
||||
|
||||
### issuerUrl
|
||||
|
||||
```ts
|
||||
issuerUrl: string;
|
||||
```
|
||||
|
||||
OIDC issuer URL or OAuth authority URL.
|
||||
For Azure: `https://login.microsoftonline.com/{tenant_id}/v2.0`
|
||||
|
||||
***
|
||||
|
||||
### managedIdentityClientId?
|
||||
|
||||
```ts
|
||||
optional managedIdentityClientId: string;
|
||||
```
|
||||
|
||||
Client ID for user-assigned managed identity (azure_managed_identity).
|
||||
|
||||
***
|
||||
|
||||
### refreshBufferSecs?
|
||||
|
||||
```ts
|
||||
optional refreshBufferSecs: number;
|
||||
```
|
||||
|
||||
Seconds before expiry to trigger proactive refresh (default: 300).
|
||||
Keep this well below the token TTL; if it is greater than or equal to
|
||||
the TTL, each request refreshes the token.
|
||||
|
||||
***
|
||||
|
||||
### scopes
|
||||
|
||||
```ts
|
||||
scopes: string[];
|
||||
```
|
||||
|
||||
OAuth scopes to request. For Azure managed identity, exactly one scope
|
||||
or resource is required. For example: `["api://{app_id}/.default"]`
|
||||
@@ -1,111 +0,0 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / OAuthConfig
|
||||
|
||||
# Interface: OAuthConfig
|
||||
|
||||
OAuth configuration for LanceDB authentication.
|
||||
|
||||
This is the public TypeScript OAuth configuration type. The generated
|
||||
`NativeOAuthConfig` type has the same runtime shape but is an implementation
|
||||
detail of the napi-rs binding.
|
||||
|
||||
All token acquisition and refresh is handled in the Rust layer.
|
||||
This config is passed through to Rust via napi-rs.
|
||||
|
||||
## Examples
|
||||
|
||||
```typescript
|
||||
const config: OAuthConfig = {
|
||||
issuerUrl: "https://login.microsoftonline.com/{tenant}/v2.0",
|
||||
clientId: "app-id",
|
||||
clientSecret: "secret",
|
||||
scopes: ["api://lancedb-api/.default"],
|
||||
};
|
||||
```
|
||||
|
||||
```typescript
|
||||
const config: OAuthConfig = {
|
||||
issuerUrl: "https://login.microsoftonline.com/{tenant}/v2.0",
|
||||
clientId: "app-id",
|
||||
scopes: ["api://lancedb-api/.default"],
|
||||
flow: OAuthFlowType.AzureManagedIdentity,
|
||||
};
|
||||
```
|
||||
|
||||
## Properties
|
||||
|
||||
### clientId
|
||||
|
||||
```ts
|
||||
clientId: string;
|
||||
```
|
||||
|
||||
Application / Client ID.
|
||||
|
||||
***
|
||||
|
||||
### clientSecret?
|
||||
|
||||
```ts
|
||||
optional clientSecret: string;
|
||||
```
|
||||
|
||||
Client secret (required for ClientCredentials).
|
||||
|
||||
***
|
||||
|
||||
### flow?
|
||||
|
||||
```ts
|
||||
optional flow: OAuthFlowType;
|
||||
```
|
||||
|
||||
Authentication flow (default: ClientCredentials).
|
||||
|
||||
***
|
||||
|
||||
### issuerUrl
|
||||
|
||||
```ts
|
||||
issuerUrl: string;
|
||||
```
|
||||
|
||||
OIDC issuer URL or OAuth authority URL.
|
||||
For Azure: `https://login.microsoftonline.com/{tenant_id}/v2.0`
|
||||
|
||||
***
|
||||
|
||||
### managedIdentityClientId?
|
||||
|
||||
```ts
|
||||
optional managedIdentityClientId: string;
|
||||
```
|
||||
|
||||
Client ID for user-assigned managed identity (AzureManagedIdentity).
|
||||
|
||||
***
|
||||
|
||||
### refreshBufferSecs?
|
||||
|
||||
```ts
|
||||
optional refreshBufferSecs: number;
|
||||
```
|
||||
|
||||
Seconds before expiry to trigger proactive refresh (default: 300).
|
||||
Keep this well below the token TTL; if it is greater than or equal to
|
||||
the TTL, each request refreshes the token.
|
||||
|
||||
***
|
||||
|
||||
### scopes
|
||||
|
||||
```ts
|
||||
scopes: string[];
|
||||
```
|
||||
|
||||
OAuth scopes to request.
|
||||
For Azure managed identity, exactly one scope or resource is required.
|
||||
For example: `["api://{app_id}/.default"]`
|
||||
@@ -8,18 +8,6 @@
|
||||
|
||||
## Properties
|
||||
|
||||
### branch?
|
||||
|
||||
```ts
|
||||
optional branch: string;
|
||||
```
|
||||
|
||||
Open the table scoped to this branch instead of the default branch.
|
||||
|
||||
Reads and writes on the returned table operate in the branch's context.
|
||||
|
||||
***
|
||||
|
||||
### ~~indexCacheSize?~~
|
||||
|
||||
```ts
|
||||
@@ -55,17 +43,3 @@ Options already set on the connection will be inherited by the table,
|
||||
but can be overridden here.
|
||||
|
||||
The available options are described at https://docs.lancedb.com/storage/
|
||||
|
||||
***
|
||||
|
||||
### version?
|
||||
|
||||
```ts
|
||||
optional version: number;
|
||||
```
|
||||
|
||||
Open the table pinned to this version, producing a read-only view.
|
||||
|
||||
Composes with [OpenTableOptions.branch](OpenTableOptions.md#branch): when both are set, opens
|
||||
that branch at the version; otherwise opens `main` at the version. Call
|
||||
`checkoutLatest` to return to a writable state.
|
||||
|
||||
@@ -8,14 +8,6 @@
|
||||
|
||||
## Properties
|
||||
|
||||
### clumpSize?
|
||||
|
||||
```ts
|
||||
optional clumpSize: number;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### counts?
|
||||
|
||||
```ts
|
||||
|
||||
@@ -1,109 +0,0 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / TokenizeOptions
|
||||
|
||||
# Interface: TokenizeOptions
|
||||
|
||||
Options for tokenizing a full-text search query without a table index.
|
||||
|
||||
## Properties
|
||||
|
||||
### asciiFolding?
|
||||
|
||||
```ts
|
||||
optional asciiFolding: boolean;
|
||||
```
|
||||
|
||||
Whether to fold ASCII characters.
|
||||
|
||||
***
|
||||
|
||||
### baseTokenizer?
|
||||
|
||||
```ts
|
||||
optional baseTokenizer: BaseTokenizer;
|
||||
```
|
||||
|
||||
The tokenizer to use. The default is "simple".
|
||||
|
||||
***
|
||||
|
||||
### language?
|
||||
|
||||
```ts
|
||||
optional language: string;
|
||||
```
|
||||
|
||||
Language for stemming and stop words.
|
||||
|
||||
***
|
||||
|
||||
### lowercase?
|
||||
|
||||
```ts
|
||||
optional lowercase: boolean;
|
||||
```
|
||||
|
||||
Whether to lowercase tokens.
|
||||
|
||||
***
|
||||
|
||||
### maxTokenLength?
|
||||
|
||||
```ts
|
||||
optional maxTokenLength: number;
|
||||
```
|
||||
|
||||
Maximum token length; tokens longer than this are ignored.
|
||||
|
||||
***
|
||||
|
||||
### ngramMaxLength?
|
||||
|
||||
```ts
|
||||
optional ngramMaxLength: number;
|
||||
```
|
||||
|
||||
N-gram maximum length.
|
||||
|
||||
***
|
||||
|
||||
### ngramMinLength?
|
||||
|
||||
```ts
|
||||
optional ngramMinLength: number;
|
||||
```
|
||||
|
||||
N-gram minimum length.
|
||||
|
||||
***
|
||||
|
||||
### prefixOnly?
|
||||
|
||||
```ts
|
||||
optional prefixOnly: boolean;
|
||||
```
|
||||
|
||||
Whether to only emit token prefixes for the n-gram tokenizer.
|
||||
|
||||
***
|
||||
|
||||
### removeStopWords?
|
||||
|
||||
```ts
|
||||
optional removeStopWords: boolean;
|
||||
```
|
||||
|
||||
Whether to remove stop words.
|
||||
|
||||
***
|
||||
|
||||
### stem?
|
||||
|
||||
```ts
|
||||
optional stem: boolean;
|
||||
```
|
||||
|
||||
Whether to stem tokens.
|
||||
@@ -1,11 +0,0 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / AnalyzePlanDistributedMetrics
|
||||
|
||||
# Type Alias: AnalyzePlanDistributedMetrics
|
||||
|
||||
```ts
|
||||
type AnalyzePlanDistributedMetrics: "aggregate" | "per_worker" | "full";
|
||||
```
|
||||
@@ -1,19 +0,0 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / BaseTokenizer
|
||||
|
||||
# Type Alias: BaseTokenizer
|
||||
|
||||
```ts
|
||||
type BaseTokenizer:
|
||||
| "simple"
|
||||
| "whitespace"
|
||||
| "raw"
|
||||
| "ngram"
|
||||
| "icu"
|
||||
| "icu/split"
|
||||
| `jieba/${string}`
|
||||
| `lindera/${string}`;
|
||||
```
|
||||
@@ -1,11 +0,0 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / TokenizeTableOptions
|
||||
|
||||
# Type Alias: TokenizeTableOptions
|
||||
|
||||
```ts
|
||||
type TokenizeTableOptions: object | object;
|
||||
```
|
||||
@@ -8,7 +8,7 @@
|
||||
<parent>
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-parent</artifactId>
|
||||
<version>0.33.0-beta.0</version>
|
||||
<version>0.30.1-beta.2</version>
|
||||
<relativePath>../pom.xml</relativePath>
|
||||
</parent>
|
||||
|
||||
|
||||
+2
-2
@@ -6,7 +6,7 @@
|
||||
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-parent</artifactId>
|
||||
<version>0.33.0-beta.0</version>
|
||||
<version>0.30.1-beta.2</version>
|
||||
<packaging>pom</packaging>
|
||||
<name>${project.artifactId}</name>
|
||||
<description>LanceDB Java SDK Parent POM</description>
|
||||
@@ -28,7 +28,7 @@
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<arrow.version>15.0.0</arrow.version>
|
||||
<lance-core.version>9.0.0</lance-core.version>
|
||||
<lance-core.version>7.2.0-beta.1</lance-core.version>
|
||||
<spotless.skip>false</spotless.skip>
|
||||
<spotless.version>2.30.0</spotless.version>
|
||||
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
|
||||
|
||||
+3
-7
@@ -1,7 +1,7 @@
|
||||
[package]
|
||||
name = "lancedb-nodejs"
|
||||
edition.workspace = true
|
||||
version = "0.33.0-beta.0"
|
||||
version = "0.30.1-beta.2"
|
||||
publish = false
|
||||
license.workspace = true
|
||||
description.workspace = true
|
||||
@@ -25,12 +25,8 @@ lancedb = { path = "../rust/lancedb", default-features = false }
|
||||
lance-namespace.workspace = true
|
||||
napi = { version = "3.8.3", default-features = false, features = [
|
||||
"napi9",
|
||||
"async",
|
||||
"chrono_date",
|
||||
"serde-json",
|
||||
"async"
|
||||
] }
|
||||
chrono = { version = "0.4", default-features = false, features = ["clock"] }
|
||||
serde_json = "1"
|
||||
napi-derive = "3.5.2"
|
||||
# Prevent dynamic linking of lzma, which comes from datafusion
|
||||
lzma-sys = { version = "0.1", features = ["static"] }
|
||||
@@ -44,6 +40,6 @@ aws-lc-rs = "=1.16.3"
|
||||
napi-build = "2.3.1"
|
||||
|
||||
[features]
|
||||
default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface", "lancedb/goosefs", "lancedb/metrics-otel"]
|
||||
default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface"]
|
||||
fp16kernels = ["lancedb/fp16kernels"]
|
||||
remote = ["lancedb/remote"]
|
||||
|
||||
@@ -52,7 +52,6 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
Float64,
|
||||
Struct,
|
||||
List,
|
||||
Map_,
|
||||
Int16,
|
||||
Int32,
|
||||
Int64,
|
||||
@@ -70,30 +69,6 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
type Schema = ApacheArrow["Schema"];
|
||||
type Table = ApacheArrow["Table"];
|
||||
|
||||
function expectValidMapField(
|
||||
// biome-ignore lint/suspicious/noExplicitAny: Arrow Field types vary across supported versions
|
||||
field: any,
|
||||
): void {
|
||||
expect(DataType.isMap(field.type)).toBe(true);
|
||||
expect(field.type.keysSorted).toBe(true);
|
||||
expect(field.type.children).toHaveLength(1);
|
||||
|
||||
const entries = field.type.children[0];
|
||||
expect(entries.name).toBe("entries");
|
||||
expect(entries.nullable).toBe(false);
|
||||
expect(DataType.isStruct(entries.type)).toBe(true);
|
||||
expect(entries.type.children).toHaveLength(2);
|
||||
|
||||
const [key, value] = entries.type.children;
|
||||
expect([key.name, value.name]).toEqual(["key", "value"]);
|
||||
expect(key.nullable).toBe(false);
|
||||
expect(DataType.isUtf8(key.type)).toBe(true);
|
||||
expect(value.nullable).toBe(true);
|
||||
expect(DataType.isInt(value.type)).toBe(true);
|
||||
expect(value.type.bitWidth).toBe(32);
|
||||
expect(value.type.isSigned).toBe(true);
|
||||
}
|
||||
|
||||
// Helper method to verify various ways to create a table
|
||||
async function checkTableCreation(
|
||||
tableCreationMethod: (
|
||||
@@ -963,34 +938,6 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it("will make an empty table with a Map field", async function () {
|
||||
const schema = new Schema([
|
||||
new Field(
|
||||
"attributes",
|
||||
new Map_(
|
||||
new Field(
|
||||
"entries",
|
||||
new Struct([
|
||||
new Field("key", new Utf8(), false),
|
||||
new Field("value", new Int32(), true),
|
||||
]),
|
||||
false,
|
||||
),
|
||||
true,
|
||||
),
|
||||
),
|
||||
]);
|
||||
|
||||
const table = makeEmptyTable(schema);
|
||||
|
||||
expectValidMapField(table.schema.fields[0]);
|
||||
|
||||
const buffer = await fromTableToBuffer(table);
|
||||
const roundTripped = tableFromIPC(buffer);
|
||||
|
||||
expectValidMapField(roundTripped.schema.fields[0]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("when using two versions of arrow", function () {
|
||||
|
||||
@@ -1,114 +0,0 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import {
|
||||
MeterProvider,
|
||||
type MetricData,
|
||||
MetricReader,
|
||||
} from "@opentelemetry/sdk-metrics";
|
||||
import * as tmp from "tmp";
|
||||
import { connect, instrumentLanceDbMetrics } from "../lancedb";
|
||||
// snapshotLancedbMetrics is internal plumbing (not part of the public API), so
|
||||
// it is imported from the native module rather than the package entry point.
|
||||
import { snapshotLancedbMetrics } from "../lancedb/native";
|
||||
|
||||
// The metrics recorder is process-global and installed once, so the whole
|
||||
// bridge is exercised in a single test to avoid cross-test global-state coupling.
|
||||
|
||||
// A minimal pull-based reader whose `collect()` we drive directly, invoking the
|
||||
// observable-instrument callbacks. `@opentelemetry/sdk-metrics` ships no
|
||||
// in-memory reader, so we subclass the abstract base.
|
||||
class TestMetricReader extends MetricReader {
|
||||
protected async onForceFlush(): Promise<void> {
|
||||
// no-op: collection is driven directly via collect()
|
||||
}
|
||||
protected async onShutdown(): Promise<void> {
|
||||
// no-op: nothing to release
|
||||
}
|
||||
}
|
||||
|
||||
async function metricsByName(
|
||||
reader: TestMetricReader,
|
||||
): Promise<Map<string, MetricData>> {
|
||||
const collected = await reader.collect();
|
||||
const result = new Map<string, MetricData>();
|
||||
for (const scope of collected.resourceMetrics.scopeMetrics) {
|
||||
for (const metric of scope.metrics) {
|
||||
result.set(metric.descriptor.name, metric);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
describe("OpenTelemetry metrics bridge", () => {
|
||||
let tmpDir: tmp.DirResult;
|
||||
beforeEach(() => {
|
||||
tmpDir = tmp.dirSync({ unsafeCleanup: true });
|
||||
});
|
||||
afterEach(() => tmpDir.removeCallback());
|
||||
|
||||
it("snapshot is safe to call regardless of install state", () => {
|
||||
expect(Array.isArray(snapshotLancedbMetrics())).toBe(true);
|
||||
});
|
||||
|
||||
it("exports object store metrics via observable instruments", async () => {
|
||||
const reader = new TestMetricReader();
|
||||
const provider = new MeterProvider({ readers: [reader] });
|
||||
expect(instrumentLanceDbMetrics(provider)).toBe(true);
|
||||
|
||||
// Generate object store activity on the local filesystem (scheme "file").
|
||||
const db = await connect(tmpDir.name);
|
||||
const data = Array.from({ length: 256 }, (_, i) => ({ id: i }));
|
||||
const table = await db.createTable("t", data);
|
||||
expect(await table.countRows()).toBe(256);
|
||||
|
||||
const metrics = await metricsByName(reader);
|
||||
|
||||
const requests = metrics.get("lance_object_store_requests_total");
|
||||
expect(requests).toBeDefined();
|
||||
// biome-ignore lint/suspicious/noExplicitAny: SDK point shape
|
||||
const requestPoints = (requests!.dataPoints as any[]) ?? [];
|
||||
expect(requestPoints.length).toBeGreaterThan(0);
|
||||
for (const p of requestPoints) {
|
||||
// Labelled by `operation` and `base` (the store scheme by default).
|
||||
expect(p.attributes).toHaveProperty("base");
|
||||
expect(p.attributes).toHaveProperty("operation");
|
||||
}
|
||||
const totalRequests = requestPoints.reduce((acc, p) => acc + p.value, 0);
|
||||
expect(totalRequests).toBeGreaterThan(0);
|
||||
|
||||
// Histograms are decomposed into bucket / count / sum observable counters.
|
||||
const bucket = metrics.get(
|
||||
"lance_object_store_request_duration_seconds_bucket",
|
||||
);
|
||||
expect(bucket).toBeDefined();
|
||||
// biome-ignore lint/suspicious/noExplicitAny: SDK point shape
|
||||
const bucketPoints = (bucket!.dataPoints as any[]) ?? [];
|
||||
expect(bucketPoints.length).toBeGreaterThan(0);
|
||||
expect(bucketPoints.every((p) => "le" in p.attributes)).toBe(true);
|
||||
// The implicit +Inf bucket must be present.
|
||||
expect(bucketPoints.some((p) => p.attributes.le === "+Inf")).toBe(true);
|
||||
|
||||
const count = metrics.get(
|
||||
"lance_object_store_request_duration_seconds_count",
|
||||
);
|
||||
expect(count).toBeDefined();
|
||||
// biome-ignore lint/suspicious/noExplicitAny: SDK point shape
|
||||
const countPoints = (count!.dataPoints as any[]) ?? [];
|
||||
expect(countPoints.reduce((acc, p) => acc + p.value, 0)).toBeGreaterThan(0);
|
||||
|
||||
const sum = metrics.get("lance_object_store_request_duration_seconds_sum");
|
||||
expect(sum).toBeDefined();
|
||||
// biome-ignore lint/suspicious/noExplicitAny: SDK point shape
|
||||
const sumPoints = (sum!.dataPoints as any[]) ?? [];
|
||||
expect(sumPoints.reduce((acc, p) => acc + p.value, 0)).toBeGreaterThan(0);
|
||||
|
||||
// Unit handling: only `_sum` keeps the histogram's unit (seconds); `_bucket`
|
||||
// and `_count` observe cumulative counts and are unitless.
|
||||
expect(sum!.descriptor.unit).toBe("s");
|
||||
expect(bucket!.descriptor.unit).toBe("");
|
||||
expect(count!.descriptor.unit).toBe("");
|
||||
|
||||
await provider.shutdown();
|
||||
});
|
||||
});
|
||||
@@ -215,20 +215,6 @@ describe("Query orderBy", () => {
|
||||
expect(results[2].score).toBeCloseTo(4.1, 0.001);
|
||||
});
|
||||
|
||||
it("should combine repeated where clauses with AND", async () => {
|
||||
const results = await table
|
||||
.query()
|
||||
.where("score > 1.0")
|
||||
.where("score < 3.0")
|
||||
.orderBy({ columnName: "score" })
|
||||
.toArray();
|
||||
// Only rows matching both predicates should be returned, rather than the
|
||||
// second where() silently replacing the first.
|
||||
expect(results.length).toBe(2);
|
||||
expect(results[0].score).toBeCloseTo(1.2, 0.001);
|
||||
expect(results[1].score).toBeCloseTo(2.8, 0.001);
|
||||
});
|
||||
|
||||
it("should support method chaining with limit", async () => {
|
||||
const results = await table
|
||||
.query()
|
||||
|
||||
@@ -191,147 +191,6 @@ describe("remote connection", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("supports version time-travel and branches on remote", async () => {
|
||||
await withMockDatabase(
|
||||
(req, res) => {
|
||||
const body = req.url?.includes("/branches/list")
|
||||
? JSON.stringify({
|
||||
branches: {
|
||||
exp: { parentVersion: 1, createAt: 1, manifestSize: 1 },
|
||||
},
|
||||
})
|
||||
: JSON.stringify({ name: "t", version: 2, schema: { fields: [] } });
|
||||
res.writeHead(200, { "Content-Type": "application/json" }).end(body);
|
||||
},
|
||||
async (db) => {
|
||||
// version-only (and "main" + version) time-travel the main chain
|
||||
const v2 = await db.openTable("t", undefined, { version: 2 });
|
||||
expect(v2.currentBranch()).toBeNull();
|
||||
const mainV2 = await db.openTable("t", undefined, {
|
||||
branch: "main",
|
||||
version: 2,
|
||||
});
|
||||
expect(mainV2.currentBranch()).toBeNull();
|
||||
|
||||
// a non-main branch opens a handle scoped to that branch
|
||||
const exp = await db.openTable("t", undefined, { branch: "exp" });
|
||||
expect(exp.currentBranch()).toBe("exp");
|
||||
const expV2 = await db.openTable("t", undefined, {
|
||||
branch: "exp",
|
||||
version: 2,
|
||||
});
|
||||
expect(expV2.currentBranch()).toBe("exp");
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("diffs and merges remote branches", async () => {
|
||||
const sampleDiff = {
|
||||
fromBranch: "exp",
|
||||
parentVersion: 1,
|
||||
mainVersion: 2,
|
||||
branchVersion: 3,
|
||||
baseMoved: false,
|
||||
rowCountMain: 3,
|
||||
rowCountBranch: 3,
|
||||
rowSummary: {
|
||||
unchanged: 3,
|
||||
newOnBase: 0,
|
||||
newOnBranch: 0,
|
||||
staleRecompute: 0,
|
||||
inputsChanged: 0,
|
||||
deltaAvailable: false,
|
||||
},
|
||||
addedColumns: [{ name: "tag", dataType: "utf8", nullable: true }],
|
||||
removedColumns: [],
|
||||
changedColumns: [],
|
||||
addedIndexes: [],
|
||||
removedIndexes: [],
|
||||
mergeable: true,
|
||||
mergeBlockers: [],
|
||||
};
|
||||
const mergeBodies: Record<string, unknown>[] = [];
|
||||
|
||||
await withMockDatabase(
|
||||
(req, res) => {
|
||||
const path = req.url ?? "";
|
||||
if (path.endsWith("/describe/")) {
|
||||
res.writeHead(200, { "Content-Type": "application/json" }).end(
|
||||
JSON.stringify({
|
||||
name: "t",
|
||||
version: 2,
|
||||
schema: { fields: [] },
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let raw = "";
|
||||
req.on("data", (chunk) => {
|
||||
raw += chunk;
|
||||
});
|
||||
req.on("end", () => {
|
||||
const body = raw ? JSON.parse(raw) : {};
|
||||
if (path.endsWith("/branches/diff/")) {
|
||||
// biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
|
||||
expect(body).toEqual({ from_branch: "exp" });
|
||||
res
|
||||
.writeHead(200, { "Content-Type": "application/json" })
|
||||
.end(JSON.stringify(sampleDiff));
|
||||
return;
|
||||
}
|
||||
if (path.endsWith("/branches/merge/")) {
|
||||
mergeBodies.push(body);
|
||||
const dryRun = body["dry_run"] === true;
|
||||
const response = {
|
||||
status: dryRun ? "ready" : "rejected",
|
||||
diff: dryRun
|
||||
? sampleDiff
|
||||
: {
|
||||
...sampleDiff,
|
||||
mergeable: false,
|
||||
mergeBlockers: [
|
||||
{ code: "baseMoved", message: "main has advanced" },
|
||||
],
|
||||
},
|
||||
preview: { promotedColumns: dryRun ? ["tag"] : [] },
|
||||
};
|
||||
res
|
||||
.writeHead(dryRun ? 200 : 409, {
|
||||
"Content-Type": "application/json",
|
||||
})
|
||||
.end(JSON.stringify(response));
|
||||
return;
|
||||
}
|
||||
res.writeHead(404).end();
|
||||
});
|
||||
},
|
||||
async (db) => {
|
||||
const table = await db.openTable("t");
|
||||
const branches = await table.branches();
|
||||
|
||||
await expect(branches.diff("exp")).resolves.toEqual(sampleDiff);
|
||||
|
||||
const rejected = await branches.merge("exp");
|
||||
expect(rejected.status).toBe("rejected");
|
||||
expect(rejected.diff.mergeBlockers).toEqual([
|
||||
{ code: "baseMoved", message: "main has advanced" },
|
||||
]);
|
||||
|
||||
const preview = await branches.merge("exp", true);
|
||||
expect(preview.status).toBe("ready");
|
||||
expect(preview.preview.promotedColumns).toEqual(["tag"]);
|
||||
},
|
||||
);
|
||||
|
||||
expect(mergeBodies).toEqual([
|
||||
// biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
|
||||
{ from_branch: "exp", dry_run: false },
|
||||
// biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
|
||||
{ from_branch: "exp", dry_run: true },
|
||||
]);
|
||||
});
|
||||
|
||||
describe("TlsConfig", () => {
|
||||
it("should create TlsConfig with all fields", () => {
|
||||
const tlsConfig: TlsConfig = {
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import * as arrow from "../lancedb/arrow";
|
||||
import { sanitizeField, sanitizeMap, sanitizeType } from "../lancedb/sanitize";
|
||||
import { sanitizeField, sanitizeType } from "../lancedb/sanitize";
|
||||
|
||||
describe("sanitize", function () {
|
||||
describe("sanitizeType function", function () {
|
||||
@@ -181,15 +181,4 @@ describe("sanitize", function () {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("sanitizeMap function", function () {
|
||||
it.each([
|
||||
["no children", []],
|
||||
["two children", [{}, {}]],
|
||||
])("should reject a Map type with %s", function (_, children) {
|
||||
expect(() => sanitizeMap({ children, keysSorted: false })).toThrow(
|
||||
"Expected a Map type to have exactly one child",
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
+25
-491
@@ -16,7 +16,6 @@ import {
|
||||
PhraseQuery,
|
||||
Table,
|
||||
connect,
|
||||
tokenize,
|
||||
} from "../lancedb";
|
||||
import {
|
||||
Table as ArrowTable,
|
||||
@@ -86,140 +85,6 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
await expect(table.countRows()).resolves.toBe(3);
|
||||
});
|
||||
|
||||
it("should support branches", async () => {
|
||||
await table.add([{ id: 1 }]);
|
||||
expect(await table.countRows()).toBe(1);
|
||||
|
||||
expect(table.currentBranch()).toBeNull();
|
||||
|
||||
// fork an isolated, writable branch from main
|
||||
const branch = await (await table.branches()).create("exp");
|
||||
expect(branch.currentBranch()).toBe("exp");
|
||||
expect(await branch.countRows()).toBe(1);
|
||||
await branch.add([{ id: 2 }]);
|
||||
expect(await branch.countRows()).toBe(2);
|
||||
// main is untouched by branch writes
|
||||
expect(await table.countRows()).toBe(1);
|
||||
|
||||
// listed, with main (null) as the parent
|
||||
const list = await (await table.branches()).list();
|
||||
expect(Object.keys(list)).toContain("exp");
|
||||
expect(list["exp"].parentBranch).toBeNull();
|
||||
|
||||
// fromRef="main" is equivalent to the default
|
||||
await (await table.branches()).create("exp2", "main");
|
||||
const list2 = await (await table.branches()).list();
|
||||
expect(list2["exp2"].parentBranch).toBeNull();
|
||||
|
||||
// checkout returns a handle scoped to the branch's latest
|
||||
const checkedOut = await (await table.branches()).checkout("exp");
|
||||
expect(checkedOut.currentBranch()).toBe("exp");
|
||||
expect(await checkedOut.countRows()).toBe(2);
|
||||
|
||||
// delete removes it
|
||||
await (await table.branches()).delete("exp");
|
||||
await (await table.branches()).delete("exp2");
|
||||
const after = await (await table.branches()).list();
|
||||
expect(Object.keys(after)).not.toContain("exp");
|
||||
});
|
||||
|
||||
it("should open a branch via open_table", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
await table.add([{ id: 1 }]);
|
||||
const branch = await (await table.branches()).create("exp");
|
||||
await branch.add([{ id: 2 }]);
|
||||
|
||||
// open_table(..., { branch }) returns a handle scoped to the branch
|
||||
const opened = await db.openTable("some_table", undefined, {
|
||||
branch: "exp",
|
||||
});
|
||||
expect(await opened.countRows()).toBe(2);
|
||||
// opening without branch still tracks main
|
||||
expect(await (await db.openTable("some_table")).countRows()).toBe(1);
|
||||
});
|
||||
|
||||
it("should open a branch at a version isolated from main and HEAD", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
// main: a single fork-point row
|
||||
const t = await db.createTable("bv_table", [{ id: 0 }]);
|
||||
const mainV1 = await t.version();
|
||||
|
||||
// fork "exp", then advance exp AND main independently past the fork so
|
||||
// they diverge while sharing version numbers
|
||||
const exp = await (await t.branches()).create("exp");
|
||||
await exp.add([{ id: 1 }]); // exp: {0, 1}
|
||||
const expV2 = await exp.version();
|
||||
await exp.add([{ id: 2 }]); // exp HEAD: {0, 1, 2}
|
||||
await t.add([{ id: 100 }, { id: 101 }, { id: 102 }]); // main HEAD: {0,100,101,102}
|
||||
expect(await t.version()).toBe(expV2);
|
||||
|
||||
// open exp at the shared version: the data must be exp's, not main's.
|
||||
// count alone cannot prove this (main@v2 also exists), so assert
|
||||
// provenance by content.
|
||||
const pinned = await db.openTable("bv_table", undefined, {
|
||||
branch: "exp",
|
||||
version: expV2,
|
||||
});
|
||||
expect(await pinned.countRows()).toBe(2); // not exp HEAD (3), not main@v2 (4)
|
||||
expect(await pinned.countRows("id = 1")).toBe(1); // exp's post-fork row
|
||||
expect(await pinned.countRows("id = 100")).toBe(0); // main's rows invisible
|
||||
|
||||
// the same coordinate is reachable directly via branches().checkout(name, version)
|
||||
const pinnedDirect = await (await t.branches()).checkout("exp", expV2);
|
||||
expect(await pinnedDirect.countRows()).toBe(2);
|
||||
|
||||
// the HEADs are unaffected
|
||||
expect(
|
||||
await (
|
||||
await db.openTable("bv_table", undefined, { branch: "exp" })
|
||||
).countRows(),
|
||||
).toBe(3);
|
||||
expect(await (await db.openTable("bv_table")).countRows()).toBe(4);
|
||||
|
||||
// version-only (no branch) time-travels main itself: its fork-point
|
||||
// version holds only main's first row, and the shared version number
|
||||
// resolves to main's data, not the branch's ("opens main at the version")
|
||||
const oldMain = await db.openTable("bv_table", undefined, {
|
||||
version: mainV1,
|
||||
});
|
||||
expect(await oldMain.countRows()).toBe(1);
|
||||
const sharedOnMain = await db.openTable("bv_table", undefined, {
|
||||
version: expV2,
|
||||
});
|
||||
expect(await sharedOnMain.countRows()).toBe(4); // main@v2, not exp@v2 (2)
|
||||
|
||||
// detached head: writing to a pinned version is rejected
|
||||
await expect(pinned.add([{ id: 9 }])).rejects.toThrow(
|
||||
/cannot be modified/,
|
||||
);
|
||||
|
||||
// a nonexistent version is rejected -- on main, and on a branch (a
|
||||
// distinct resolution path, on the branch's manifests)
|
||||
await expect(
|
||||
db.openTable("bv_table", undefined, { version: 9999 }),
|
||||
).rejects.toThrow();
|
||||
await expect(
|
||||
db.openTable("bv_table", undefined, { branch: "exp", version: 9999 }),
|
||||
).rejects.toThrow();
|
||||
|
||||
// checkoutLatest re-attaches the pinned handle to the BRANCH's HEAD
|
||||
// (writable again), not main's HEAD (4), and not staying pinned (2)
|
||||
await pinned.checkoutLatest();
|
||||
expect(await pinned.countRows()).toBe(3); // exp HEAD
|
||||
await pinned.add([{ id: 3 }]);
|
||||
expect(await pinned.countRows()).toBe(4); // writable again
|
||||
});
|
||||
|
||||
it("rejects invalid branch inputs", async () => {
|
||||
const branches = await table.branches();
|
||||
await expect(branches.create("")).rejects.toThrow("non-empty");
|
||||
await expect(branches.checkout("")).rejects.toThrow("non-empty");
|
||||
await expect(branches.delete("")).rejects.toThrow("non-empty");
|
||||
await expect(branches.create("bad", "main", -1)).rejects.toThrow(
|
||||
"non-negative",
|
||||
);
|
||||
});
|
||||
|
||||
it("should show table stats", async () => {
|
||||
await table.add([{ id: 1 }, { id: 2 }]);
|
||||
await table.add([{ id: 1 }]);
|
||||
@@ -850,15 +715,13 @@ describe("When creating an index", () => {
|
||||
expect(fs.readdirSync(indexDir)).toHaveLength(1);
|
||||
const indices = await tbl.listIndices();
|
||||
expect(indices.length).toBe(1);
|
||||
expect(indices[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
name: "vec_idx",
|
||||
indexType: "IvfPq",
|
||||
columns: ["vec"],
|
||||
}),
|
||||
);
|
||||
expect(indices[0]).toEqual({
|
||||
name: "vec_idx",
|
||||
indexType: "IvfPq",
|
||||
columns: ["vec"],
|
||||
});
|
||||
const stats = await tbl.indexStats("vec_idx");
|
||||
expect(stats).toBeDefined();
|
||||
expect(stats?.loss).toBeDefined();
|
||||
|
||||
// Search without specifying the column
|
||||
let rst = await tbl
|
||||
@@ -918,22 +781,10 @@ describe("When creating an index", () => {
|
||||
expect(indices2.length).toBe(0);
|
||||
});
|
||||
|
||||
it("should preserve canonical nested field paths across index lifecycle", async () => {
|
||||
it("should create and search a nested vector index", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
const nestedSchema = new Schema([
|
||||
new Field("rowId", new Int32(), true),
|
||||
new Field("row-id", new Int32(), true),
|
||||
new Field("userId", new Int32(), true),
|
||||
new Field(
|
||||
"metadata",
|
||||
new Struct([new Field("user_id", new Int32(), true)]),
|
||||
true,
|
||||
),
|
||||
new Field(
|
||||
"MetaData",
|
||||
new Struct([new Field("userId", new Int32(), true)]),
|
||||
true,
|
||||
),
|
||||
new Field("id", new Int32(), true),
|
||||
new Field(
|
||||
"image",
|
||||
new Struct([
|
||||
@@ -945,146 +796,27 @@ describe("When creating an index", () => {
|
||||
]),
|
||||
true,
|
||||
),
|
||||
new Field(
|
||||
"payload",
|
||||
new Struct([new Field("text", new Utf8(), true)]),
|
||||
true,
|
||||
),
|
||||
new Field(
|
||||
"meta-data",
|
||||
new Struct([new Field("user-id", new Int32(), true)]),
|
||||
true,
|
||||
),
|
||||
new Field(
|
||||
"literal",
|
||||
new Struct([new Field("a.b", new Int32(), true)]),
|
||||
true,
|
||||
),
|
||||
]);
|
||||
const nestedTable = await db.createTable(
|
||||
"nested_field_index_lifecycle",
|
||||
"nested_vector",
|
||||
makeArrowTable(
|
||||
Array.from({ length: 300 }, (_, rowId) => ({
|
||||
rowId,
|
||||
"row-id": rowId,
|
||||
userId: rowId,
|
||||
metadata: { ["user_id"]: rowId },
|
||||
["MetaData"]: { userId: rowId },
|
||||
image: { embedding: [rowId, rowId + 1] },
|
||||
payload: { text: `document ${rowId}` },
|
||||
"meta-data": { "user-id": rowId },
|
||||
literal: { "a.b": rowId },
|
||||
Array.from({ length: 300 }, (_, id) => ({
|
||||
id,
|
||||
image: { embedding: [id, id + 1] },
|
||||
})),
|
||||
{ schema: nestedSchema },
|
||||
),
|
||||
);
|
||||
|
||||
await nestedTable.createIndex("rowId", {
|
||||
config: Index.btree(),
|
||||
name: "row_id_idx",
|
||||
});
|
||||
await nestedTable.createIndex("`row-id`", {
|
||||
config: Index.btree(),
|
||||
name: "row_dash_id_idx",
|
||||
});
|
||||
await nestedTable.createIndex("userId", {
|
||||
config: Index.btree(),
|
||||
name: "top_user_id_idx",
|
||||
});
|
||||
await nestedTable.createIndex("metadata.user_id", {
|
||||
config: Index.btree(),
|
||||
name: "nested_user_id_idx",
|
||||
});
|
||||
await nestedTable.createIndex("MetaData.userId", {
|
||||
config: Index.btree(),
|
||||
name: "mixed_case_metadata_user_id_idx",
|
||||
});
|
||||
await nestedTable.createIndex("`meta-data`.`user-id`", {
|
||||
config: Index.btree(),
|
||||
name: "escaped_names_idx",
|
||||
});
|
||||
await nestedTable.createIndex("literal.`a.b`", {
|
||||
config: Index.btree(),
|
||||
name: "literal_dot_idx",
|
||||
});
|
||||
await nestedTable.createIndex("image.embedding", {
|
||||
name: "image_embedding_idx",
|
||||
});
|
||||
await nestedTable.createIndex("payload.text", {
|
||||
config: Index.fts({ withPosition: false }),
|
||||
name: "payload_text_idx",
|
||||
});
|
||||
|
||||
const indices = await nestedTable.listIndices();
|
||||
expect(indices).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
name: "row_id_idx",
|
||||
indexType: "BTree",
|
||||
columns: ["rowId"],
|
||||
}),
|
||||
expect.objectContaining({
|
||||
name: "row_dash_id_idx",
|
||||
indexType: "BTree",
|
||||
columns: ["`row-id`"],
|
||||
}),
|
||||
expect.objectContaining({
|
||||
name: "top_user_id_idx",
|
||||
indexType: "BTree",
|
||||
columns: ["userId"],
|
||||
}),
|
||||
expect.objectContaining({
|
||||
name: "nested_user_id_idx",
|
||||
indexType: "BTree",
|
||||
columns: ["metadata.user_id"],
|
||||
}),
|
||||
expect.objectContaining({
|
||||
name: "mixed_case_metadata_user_id_idx",
|
||||
indexType: "BTree",
|
||||
columns: ["MetaData.userId"],
|
||||
}),
|
||||
expect.objectContaining({
|
||||
name: "escaped_names_idx",
|
||||
indexType: "BTree",
|
||||
columns: ["`meta-data`.`user-id`"],
|
||||
}),
|
||||
expect.objectContaining({
|
||||
name: "literal_dot_idx",
|
||||
indexType: "BTree",
|
||||
columns: ["literal.`a.b`"],
|
||||
}),
|
||||
expect.objectContaining({
|
||||
name: "image_embedding_idx",
|
||||
indexType: "IvfPq",
|
||||
columns: ["image.embedding"],
|
||||
}),
|
||||
expect.objectContaining({
|
||||
name: "payload_text_idx",
|
||||
indexType: "FTS",
|
||||
columns: ["payload.text"],
|
||||
}),
|
||||
]),
|
||||
);
|
||||
|
||||
const stats = await nestedTable.indexStats(
|
||||
"mixed_case_metadata_user_id_idx",
|
||||
);
|
||||
expect(stats?.numIndexedRows).toEqual(300);
|
||||
expect(stats?.indexType).toEqual("BTREE");
|
||||
|
||||
const filtered = await nestedTable
|
||||
.query()
|
||||
.where("MetaData.userId = 42")
|
||||
.limit(1)
|
||||
.toArray();
|
||||
expect(filtered[0].MetaData.userId).toEqual(42);
|
||||
|
||||
const escapedFiltered = await nestedTable
|
||||
.query()
|
||||
.where("`row-id` = 43")
|
||||
.limit(1)
|
||||
.toArray();
|
||||
expect(escapedFiltered[0]["row-id"]).toEqual(43);
|
||||
expect(indices).toContainEqual({
|
||||
name: "image_embedding_idx",
|
||||
indexType: "IvfPq",
|
||||
columns: ["image.embedding"],
|
||||
});
|
||||
|
||||
const explicit = await nestedTable
|
||||
.query()
|
||||
@@ -1097,37 +829,7 @@ describe("When creating an index", () => {
|
||||
.nearestTo([0.0, 1.0])
|
||||
.limit(1)
|
||||
.toArray();
|
||||
expect(inferred[0].rowId).toEqual(explicit[0].rowId);
|
||||
|
||||
await nestedTable.add([
|
||||
{
|
||||
rowId: 300,
|
||||
"row-id": 300,
|
||||
userId: 300,
|
||||
metadata: { ["user_id"]: 300 },
|
||||
["MetaData"]: { userId: 300 },
|
||||
image: { embedding: [300.0, 301.0] },
|
||||
payload: { text: "document 300" },
|
||||
"meta-data": { "user-id": 300 },
|
||||
literal: { "a.b": 300 },
|
||||
},
|
||||
]);
|
||||
await nestedTable.optimize();
|
||||
const indicesAfterOptimize = await nestedTable.listIndices();
|
||||
expect(indicesAfterOptimize).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
name: "mixed_case_metadata_user_id_idx",
|
||||
indexType: "BTree",
|
||||
columns: ["MetaData.userId"],
|
||||
}),
|
||||
expect.objectContaining({
|
||||
name: "image_embedding_idx",
|
||||
indexType: "IvfPq",
|
||||
columns: ["image.embedding"],
|
||||
}),
|
||||
]),
|
||||
);
|
||||
expect(inferred[0].id).toEqual(explicit[0].id);
|
||||
});
|
||||
|
||||
it("should report multiple nested vector candidates", async () => {
|
||||
@@ -1261,13 +963,11 @@ describe("When creating an index", () => {
|
||||
expect(fs.readdirSync(indexDir)).toHaveLength(1);
|
||||
const indices = await tbl.listIndices();
|
||||
expect(indices.length).toBe(1);
|
||||
expect(indices[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
name: "vec_idx",
|
||||
indexType: "IvfHnswSq",
|
||||
columns: ["vec"],
|
||||
}),
|
||||
);
|
||||
expect(indices[0]).toEqual({
|
||||
name: "vec_idx",
|
||||
indexType: "IvfHnswSq",
|
||||
columns: ["vec"],
|
||||
});
|
||||
|
||||
// Search without specifying the column
|
||||
let rst = await tbl
|
||||
@@ -1440,20 +1140,6 @@ describe("When creating an index", () => {
|
||||
expect(fs.readdirSync(indexDir)).toHaveLength(1);
|
||||
});
|
||||
|
||||
test("create an FM index", async () => {
|
||||
// FM-Index accelerates substring search on a string/binary column.
|
||||
const db = await connect(tmpDir.name);
|
||||
const fmTbl = await db.createTable("fm_table", [
|
||||
{ id: 0, text: "hello world" },
|
||||
{ id: 1, text: "foo bar" },
|
||||
]);
|
||||
await fmTbl.createIndex("text", {
|
||||
config: Index.fm(),
|
||||
});
|
||||
const indexDir = path.join(tmpDir.name, "fm_table.lance", "_indices");
|
||||
expect(fs.readdirSync(indexDir)).toHaveLength(1);
|
||||
});
|
||||
|
||||
test("should be able to get index stats", async () => {
|
||||
await tbl.createIndex("id");
|
||||
|
||||
@@ -1464,6 +1150,7 @@ describe("When creating an index", () => {
|
||||
expect(stats?.distanceType).toBeUndefined();
|
||||
expect(stats?.indexType).toEqual("BTREE");
|
||||
expect(stats?.numIndices).toEqual(1);
|
||||
expect(stats?.loss).toBeUndefined();
|
||||
});
|
||||
|
||||
test("when getting stats on non-existent index", async () => {
|
||||
@@ -1613,35 +1300,6 @@ describe("When creating an index", () => {
|
||||
expect(rst64Query.toString()).toEqual(rst64Search.toString());
|
||||
expect(rst64Query.numRows).toBe(2);
|
||||
});
|
||||
|
||||
it("should expose rich metadata fields on IndexConfig", async () => {
|
||||
await tbl.createIndex("id", { config: Index.btree() });
|
||||
await tbl.createIndex("vec");
|
||||
|
||||
const indicesByName = Object.fromEntries(
|
||||
(await tbl.listIndices()).map((idx) => [idx.name, idx]),
|
||||
);
|
||||
|
||||
const scalarIdx = indicesByName["id_idx"];
|
||||
expect(scalarIdx).toBeDefined();
|
||||
expect(typeof scalarIdx.indexUuid).toBe("string");
|
||||
expect(scalarIdx.numIndexedRows).toBe(300);
|
||||
expect(scalarIdx.numUnindexedRows).toBe(0);
|
||||
expect(scalarIdx.numSegments).toBeGreaterThanOrEqual(1);
|
||||
expect(scalarIdx.sizeBytes).toBeGreaterThan(0);
|
||||
// Use toString check to avoid cross-realm instanceof failures with native Date objects
|
||||
expect(Object.prototype.toString.call(scalarIdx.createdAt)).toBe(
|
||||
"[object Date]",
|
||||
);
|
||||
expect((scalarIdx.createdAt as Date).getTime()).toBeGreaterThan(0);
|
||||
expect(typeof scalarIdx.indexDetails).toBe("object");
|
||||
|
||||
const vectorIdx = indicesByName["vec_idx"];
|
||||
expect(vectorIdx).toBeDefined();
|
||||
expect(typeof vectorIdx.indexUuid).toBe("string");
|
||||
expect(vectorIdx.numIndexedRows).toBe(300);
|
||||
expect(typeof vectorIdx.indexDetails).toBe("object");
|
||||
});
|
||||
});
|
||||
|
||||
describe("When querying a table", () => {
|
||||
@@ -2308,75 +1966,6 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
expect(results2[0].text).toBe(data[1].text);
|
||||
});
|
||||
|
||||
test("tokenizes FTS queries by column or index name", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
const data = [
|
||||
{
|
||||
text: "Running in cafés",
|
||||
japanese: "Hello, こんにちは世界!",
|
||||
vector: [0.1, 0.2, 0.3],
|
||||
},
|
||||
];
|
||||
const table = await db.createTable("test", data);
|
||||
await table.createIndex("text", {
|
||||
config: Index.fts({ baseTokenizer: "simple" }),
|
||||
});
|
||||
await table.createIndex("japanese", {
|
||||
config: Index.fts({
|
||||
baseTokenizer: "icu",
|
||||
stem: false,
|
||||
removeStopWords: false,
|
||||
}),
|
||||
name: "japanese_icu_idx",
|
||||
});
|
||||
|
||||
await expect(table.tokenize("hello", {} as never)).rejects.toThrow(
|
||||
"Specify exactly one",
|
||||
);
|
||||
await expect(
|
||||
table.tokenize("hello", {
|
||||
column: "text",
|
||||
indexName: "text_idx",
|
||||
} as never),
|
||||
).rejects.toThrow("Specify exactly one");
|
||||
|
||||
const simpleTokens = await table.tokenize("Running in cafés", {
|
||||
column: "text",
|
||||
});
|
||||
expect(simpleTokens).toEqual([
|
||||
{ text: "run", position: 0 },
|
||||
{ text: "cafe", position: 2 },
|
||||
]);
|
||||
|
||||
const icuTokens = await table.tokenize("Hello, こんにちは世界!", {
|
||||
indexName: "japanese_icu_idx",
|
||||
});
|
||||
expect(icuTokens).toEqual([
|
||||
{ text: "hello", position: 0 },
|
||||
{ text: "こんにちは", position: 1 },
|
||||
{ text: "世界", position: 2 },
|
||||
]);
|
||||
|
||||
const directSimpleTokens = await tokenize("Running in cafés", {
|
||||
baseTokenizer: "simple",
|
||||
});
|
||||
expect(directSimpleTokens).toEqual([
|
||||
{ text: "run", position: 0 },
|
||||
{ text: "cafe", position: 2 },
|
||||
]);
|
||||
|
||||
const directIcuTokens = await tokenize("Hello, こんにちは世界!", {
|
||||
baseTokenizer: "icu",
|
||||
stem: false,
|
||||
removeStopWords: false,
|
||||
});
|
||||
expect(directIcuTokens).toEqual([
|
||||
{ text: "hello", position: 0 },
|
||||
{ text: "こんにちは", position: 1 },
|
||||
{ text: "世界", position: 2 },
|
||||
]);
|
||||
});
|
||||
|
||||
test("full text search fast search", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
const data = [{ text: "hello world", vector: [0.1, 0.2, 0.3], id: 1 }];
|
||||
@@ -2775,13 +2364,8 @@ describe("when calling analyzePlan", () => {
|
||||
.fill(1)
|
||||
.map(() => Math.random());
|
||||
const plan = await table.query().nearestTo(queryVec).analyzePlan();
|
||||
console.log("Query Plan:\n", plan); // <--- Print the plan
|
||||
expect(plan).toMatch("AnalyzeExec");
|
||||
|
||||
const fullPlan = await table
|
||||
.query()
|
||||
.nearestTo(queryVec)
|
||||
.analyzePlan("full");
|
||||
expect(fullPlan).toMatch("AnalyzeExec");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -3067,56 +2651,6 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => {
|
||||
}),
|
||||
).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("reads back the installed spec via getLsmWriteSpec", async () => {
|
||||
const conn = await connect(tmpDir.name);
|
||||
const table = await makeTable(conn);
|
||||
await table.setUnenforcedPrimaryKey("id");
|
||||
|
||||
// Nothing installed yet.
|
||||
expect(await table.getLsmWriteSpec()).toBeUndefined();
|
||||
|
||||
// A real scalar index is needed to name it as a maintained index.
|
||||
await table.add([{ id: 1 }, { id: 2 }, { id: 3 }]);
|
||||
await table.createIndex("id");
|
||||
const indexName = (await table.listIndices())[0].name;
|
||||
|
||||
// Bucket spec round-trips, including maintained indexes and writer config
|
||||
// defaults. Lance writer-config keys are canonically snake_case.
|
||||
// biome-ignore lint/style/useNamingConvention: Lance writer-config keys are snake_case
|
||||
const writerConfigDefaults = { durable_write: "false" };
|
||||
await table.setLsmWriteSpec({
|
||||
specType: "bucket",
|
||||
column: "id",
|
||||
numBuckets: 4,
|
||||
maintainedIndexes: [indexName],
|
||||
writerConfigDefaults,
|
||||
});
|
||||
const spec = await table.getLsmWriteSpec();
|
||||
expect(spec).toBeDefined();
|
||||
expect(spec?.specType).toBe("bucket");
|
||||
expect(spec?.column).toBe("id");
|
||||
expect(spec?.numBuckets).toBe(4);
|
||||
expect(spec?.maintainedIndexes).toEqual([indexName]);
|
||||
expect(spec?.writerConfigDefaults).toEqual(writerConfigDefaults);
|
||||
|
||||
// After unset, undefined again.
|
||||
await table.unsetLsmWriteSpec();
|
||||
expect(await table.getLsmWriteSpec()).toBeUndefined();
|
||||
|
||||
// Identity round-trips (column recovered from the schema).
|
||||
await table.setLsmWriteSpec({ specType: "identity", column: "id" });
|
||||
const identity = await table.getLsmWriteSpec();
|
||||
expect(identity?.specType).toBe("identity");
|
||||
expect(identity?.column).toBe("id");
|
||||
await table.unsetLsmWriteSpec();
|
||||
|
||||
// Unsharded round-trips (no routing column).
|
||||
await table.setLsmWriteSpec({ specType: "unsharded" });
|
||||
const unsharded = await table.getLsmWriteSpec();
|
||||
expect(unsharded?.specType).toBe("unsharded");
|
||||
expect(unsharded?.column).toBeFalsy();
|
||||
});
|
||||
});
|
||||
|
||||
describe("LSM merge insert", () => {
|
||||
|
||||
@@ -84,20 +84,6 @@ export interface CreateTableOptions {
|
||||
}
|
||||
|
||||
export interface OpenTableOptions {
|
||||
/**
|
||||
* Open the table scoped to this branch instead of the default branch.
|
||||
*
|
||||
* Reads and writes on the returned table operate in the branch's context.
|
||||
*/
|
||||
branch?: string;
|
||||
/**
|
||||
* Open the table pinned to this version, producing a read-only view.
|
||||
*
|
||||
* Composes with {@link OpenTableOptions.branch}: when both are set, opens
|
||||
* that branch at the version; otherwise opens `main` at the version. Call
|
||||
* `checkoutLatest` to return to a writable state.
|
||||
*/
|
||||
version?: number;
|
||||
/**
|
||||
* Configuration for object storage.
|
||||
*
|
||||
@@ -497,20 +483,7 @@ export class LocalConnection extends Connection {
|
||||
options?.indexCacheSize,
|
||||
);
|
||||
|
||||
let table: Table = new LocalTable(innerTable);
|
||||
// "main" is the default branch, so treat it as no branch. On a real branch,
|
||||
// scope and pin in one step (yielding "version V of branch B"); otherwise
|
||||
// pin the version, if any, against main.
|
||||
const branch =
|
||||
options?.branch != null && options.branch !== "main"
|
||||
? options.branch
|
||||
: undefined;
|
||||
if (branch != null) {
|
||||
table = await (await table.branches()).checkout(branch, options?.version);
|
||||
} else if (options?.version != null) {
|
||||
await table.checkout(options.version);
|
||||
}
|
||||
return table;
|
||||
return new LocalTable(innerTable);
|
||||
}
|
||||
|
||||
async cloneTable(
|
||||
|
||||
@@ -13,21 +13,13 @@ import {
|
||||
Connection as LanceDbConnection,
|
||||
JsHeaderProvider as NativeJsHeaderProvider,
|
||||
Session,
|
||||
tokenize as nativeTokenize,
|
||||
} from "./native.js";
|
||||
|
||||
import { HeaderProvider } from "./header";
|
||||
import type { BaseTokenizer } from "./indices";
|
||||
import type { FtsToken } from "./table";
|
||||
|
||||
// Re-export native header provider for use with connectWithHeaderProvider
|
||||
export { JsHeaderProvider as NativeJsHeaderProvider } from "./native.js";
|
||||
|
||||
// OpenTelemetry metrics bridge. Only the high-level entry point is public; the
|
||||
// underlying recorder/catalog/snapshot functions remain internal plumbing that
|
||||
// `otel.ts` consumes from the native module.
|
||||
export { instrumentLanceDbMetrics } from "./otel";
|
||||
|
||||
export {
|
||||
AddColumnsSql,
|
||||
ConnectionOptions,
|
||||
@@ -46,7 +38,6 @@ export {
|
||||
FragmentSummaryStats,
|
||||
Tags,
|
||||
TagContents,
|
||||
BranchContents,
|
||||
MergeResult,
|
||||
AddResult,
|
||||
AddColumnsResult,
|
||||
@@ -60,7 +51,6 @@ export {
|
||||
SplitHashOptions,
|
||||
SplitSequentialOptions,
|
||||
ShuffleOptions,
|
||||
OAuthConfig as NativeOAuthConfig,
|
||||
} from "./native.js";
|
||||
|
||||
export {
|
||||
@@ -93,7 +83,6 @@ export {
|
||||
QueryBase,
|
||||
VectorQuery,
|
||||
TakeQuery,
|
||||
AnalyzePlanDistributedMetrics,
|
||||
QueryExecutionOptions,
|
||||
ColumnOrdering,
|
||||
FullTextSearchOptions,
|
||||
@@ -118,27 +107,15 @@ export {
|
||||
HnswPqOptions,
|
||||
HnswSqOptions,
|
||||
FtsOptions,
|
||||
BaseTokenizer,
|
||||
} from "./indices";
|
||||
|
||||
export {
|
||||
Table,
|
||||
Branches,
|
||||
BranchColumnSummary,
|
||||
BranchColumnChange,
|
||||
BranchIndexSummary,
|
||||
BranchRowCountSummary,
|
||||
MergeBlocker,
|
||||
BranchDiff,
|
||||
MergePreview,
|
||||
MergeBranchResult,
|
||||
AddDataOptions,
|
||||
UpdateOptions,
|
||||
OptimizeOptions,
|
||||
Version,
|
||||
WriteProgress,
|
||||
FtsToken,
|
||||
TokenizeTableOptions,
|
||||
LsmWriteSpec,
|
||||
ColumnAlteration,
|
||||
FieldMetadataUpdate,
|
||||
@@ -151,8 +128,6 @@ export {
|
||||
TokenResponse,
|
||||
} from "./header";
|
||||
|
||||
export { OAuthConfig, OAuthFlowType } from "./oauth";
|
||||
|
||||
export { MergeInsertBuilder, WriteExecutionOptions } from "./merge";
|
||||
|
||||
export * as embedding from "./embedding";
|
||||
@@ -170,68 +145,6 @@ export {
|
||||
} from "./arrow";
|
||||
export { IntoSql, packBits } from "./util";
|
||||
|
||||
/**
|
||||
* Options for tokenizing a full-text search query without a table index.
|
||||
*/
|
||||
export interface TokenizeOptions {
|
||||
/**
|
||||
* The tokenizer to use. The default is "simple".
|
||||
*/
|
||||
baseTokenizer?: BaseTokenizer;
|
||||
|
||||
/** Language for stemming and stop words. */
|
||||
language?: string;
|
||||
|
||||
/** Maximum token length; tokens longer than this are ignored. */
|
||||
maxTokenLength?: number;
|
||||
|
||||
/** Whether to lowercase tokens. */
|
||||
lowercase?: boolean;
|
||||
|
||||
/** Whether to stem tokens. */
|
||||
stem?: boolean;
|
||||
|
||||
/** Whether to remove stop words. */
|
||||
removeStopWords?: boolean;
|
||||
|
||||
/** Whether to fold ASCII characters. */
|
||||
asciiFolding?: boolean;
|
||||
|
||||
/** N-gram minimum length. */
|
||||
ngramMinLength?: number;
|
||||
|
||||
/** N-gram maximum length. */
|
||||
ngramMaxLength?: number;
|
||||
|
||||
/** Whether to only emit token prefixes for the n-gram tokenizer. */
|
||||
prefixOnly?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Tokenize a full-text search query using an explicit tokenizer.
|
||||
*
|
||||
* This does not require a table or FTS index. The tokenizer options match
|
||||
* {@link Index.fts}.
|
||||
*/
|
||||
export async function tokenize(
|
||||
query: string,
|
||||
options?: Partial<TokenizeOptions>,
|
||||
): Promise<FtsToken[]> {
|
||||
return await nativeTokenize(
|
||||
query,
|
||||
options?.baseTokenizer,
|
||||
options?.language,
|
||||
options?.maxTokenLength,
|
||||
options?.lowercase,
|
||||
options?.stem,
|
||||
options?.removeStopWords,
|
||||
options?.asciiFolding,
|
||||
options?.ngramMinLength,
|
||||
options?.ngramMaxLength,
|
||||
options?.prefixOnly,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Connect to a LanceDB instance at the given URI.
|
||||
*
|
||||
|
||||
@@ -486,16 +486,6 @@ export interface IvfFlatOptions {
|
||||
sampleRate?: number;
|
||||
}
|
||||
|
||||
export type BaseTokenizer =
|
||||
| "simple"
|
||||
| "whitespace"
|
||||
| "raw"
|
||||
| "ngram"
|
||||
| "icu"
|
||||
| "icu/split"
|
||||
| `jieba/${string}`
|
||||
| `lindera/${string}`;
|
||||
|
||||
/**
|
||||
* Options to create a full text search index
|
||||
*/
|
||||
@@ -519,12 +509,8 @@ export interface FtsOptions {
|
||||
* "whitespace" - Whitespace tokenizer. This tokenizer splits the text into tokens using whitespace as a delimiter.
|
||||
*
|
||||
* "raw" - Raw tokenizer. This tokenizer does not split the text into tokens and indexes the entire text as a single token.
|
||||
*
|
||||
* "icu" - ICU dictionary-based word segmentation.
|
||||
*
|
||||
* "icu/split" - ICU segmentation with simple-style delimiter splitting.
|
||||
*/
|
||||
baseTokenizer?: BaseTokenizer;
|
||||
baseTokenizer?: "simple" | "whitespace" | "raw" | "ngram";
|
||||
|
||||
/**
|
||||
* language for stemming and stop words
|
||||
@@ -716,17 +702,6 @@ export class Index {
|
||||
return new Index(LanceDbIndex.labelList());
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an FM-Index.
|
||||
*
|
||||
* An FM-Index is a scalar index on string or binary columns that accelerates
|
||||
* substring search, i.e. `contains(col, 'needle')`. Unlike the tokenized
|
||||
* full-text-search index, it matches arbitrary substrings of the raw bytes.
|
||||
*/
|
||||
static fm() {
|
||||
return new Index(LanceDbIndex.fm());
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a full text search index
|
||||
*
|
||||
|
||||
@@ -1,76 +0,0 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
/**
|
||||
* OAuth authentication flow types.
|
||||
*/
|
||||
export enum OAuthFlowType {
|
||||
/** Client Credentials grant (service-to-service / M2M). */
|
||||
ClientCredentials = "client_credentials",
|
||||
/** Azure Managed Identity via IMDS. */
|
||||
AzureManagedIdentity = "azure_managed_identity",
|
||||
}
|
||||
|
||||
/**
|
||||
* OAuth configuration for LanceDB authentication.
|
||||
*
|
||||
* This is the public TypeScript OAuth configuration type. The generated
|
||||
* `NativeOAuthConfig` type has the same runtime shape but is an implementation
|
||||
* detail of the napi-rs binding.
|
||||
*
|
||||
* All token acquisition and refresh is handled in the Rust layer.
|
||||
* This config is passed through to Rust via napi-rs.
|
||||
*
|
||||
* @example Client Credentials (service-to-service):
|
||||
* ```typescript
|
||||
* const config: OAuthConfig = {
|
||||
* issuerUrl: "https://login.microsoftonline.com/{tenant}/v2.0",
|
||||
* clientId: "app-id",
|
||||
* clientSecret: "secret",
|
||||
* scopes: ["api://lancedb-api/.default"],
|
||||
* };
|
||||
* ```
|
||||
*
|
||||
* @example Azure Managed Identity:
|
||||
* ```typescript
|
||||
* const config: OAuthConfig = {
|
||||
* issuerUrl: "https://login.microsoftonline.com/{tenant}/v2.0",
|
||||
* clientId: "app-id",
|
||||
* scopes: ["api://lancedb-api/.default"],
|
||||
* flow: OAuthFlowType.AzureManagedIdentity,
|
||||
* };
|
||||
* ```
|
||||
*/
|
||||
export interface OAuthConfig {
|
||||
/**
|
||||
* OIDC issuer URL or OAuth authority URL.
|
||||
* For Azure: `https://login.microsoftonline.com/{tenant_id}/v2.0`
|
||||
*/
|
||||
issuerUrl: string;
|
||||
|
||||
/** Application / Client ID. */
|
||||
clientId: string;
|
||||
|
||||
/**
|
||||
* OAuth scopes to request.
|
||||
* For Azure managed identity, exactly one scope or resource is required.
|
||||
* For example: `["api://{app_id}/.default"]`
|
||||
*/
|
||||
scopes: string[];
|
||||
|
||||
/** Authentication flow (default: ClientCredentials). */
|
||||
flow?: OAuthFlowType;
|
||||
|
||||
/** Client secret (required for ClientCredentials). */
|
||||
clientSecret?: string;
|
||||
|
||||
/** Client ID for user-assigned managed identity (AzureManagedIdentity). */
|
||||
managedIdentityClientId?: string;
|
||||
|
||||
/**
|
||||
* Seconds before expiry to trigger proactive refresh (default: 300).
|
||||
* Keep this well below the token TTL; if it is greater than or equal to
|
||||
* the TTL, each request refreshes the token.
|
||||
*/
|
||||
refreshBufferSecs?: number;
|
||||
}
|
||||
@@ -1,137 +0,0 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import {
|
||||
type Attributes,
|
||||
type MeterProvider,
|
||||
type ObservableResult,
|
||||
metrics,
|
||||
} from "@opentelemetry/api";
|
||||
|
||||
import {
|
||||
lancedbMetricsCatalog,
|
||||
registerLancedbMetricsRecorder,
|
||||
snapshotLancedbMetrics,
|
||||
} from "./native";
|
||||
|
||||
let instrumented = false;
|
||||
|
||||
/**
|
||||
* Register LanceDB metrics as OpenTelemetry observable instruments.
|
||||
*
|
||||
* Installs a process-global metrics recorder and creates one observable
|
||||
* instrument per LanceDB metric (currently object store request counts, bytes,
|
||||
* latency, errors, and throttles) on the given (or global) `MeterProvider`. The
|
||||
* configured `MetricReader` then collects them on its own schedule.
|
||||
*
|
||||
* Counters and gauges map directly to observable counters/gauges. Because
|
||||
* OpenTelemetry has no asynchronous histogram instrument, each histogram is
|
||||
* exported Prometheus-style as cumulative `le` bucket counts (`<name>_bucket`,
|
||||
* with an `le` attribute) plus `<name>_count` and `<name>_sum`.
|
||||
*
|
||||
* Requires `@opentelemetry/api` (a dependency) and, to actually export, an
|
||||
* OpenTelemetry SDK such as `@opentelemetry/sdk-metrics`.
|
||||
*
|
||||
* @param meterProvider The provider to register instruments on. Defaults to the
|
||||
* global provider from `@opentelemetry/api`.
|
||||
* @returns `true` if the recorder is installed and instruments are registered.
|
||||
* `false` if a different `metrics` recorder is already installed in this
|
||||
* process (only one global recorder is permitted), in which case a warning is
|
||||
* emitted and no instruments are created. Calling this more than once is safe;
|
||||
* instruments are created only on the first successful call.
|
||||
*/
|
||||
export function instrumentLanceDbMetrics(
|
||||
meterProvider?: MeterProvider,
|
||||
): boolean {
|
||||
if (!registerLancedbMetricsRecorder()) {
|
||||
console.warn(
|
||||
"Could not install the LanceDB metrics recorder: another `metrics` " +
|
||||
"recorder is already installed in this process. LanceDB metrics will " +
|
||||
"not be exported via OpenTelemetry.",
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (instrumented) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const provider = meterProvider ?? metrics.getMeterProvider();
|
||||
const meter = provider.getMeter("lancedb");
|
||||
|
||||
const scalarCallback = (metricName: string) => (result: ObservableResult) => {
|
||||
for (const point of snapshotLancedbMetrics()) {
|
||||
if (point.name === metricName && point.value != null) {
|
||||
result.observe(point.value, point.attributes);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const bucketCallback = (metricName: string) => (result: ObservableResult) => {
|
||||
for (const point of snapshotLancedbMetrics()) {
|
||||
if (point.name !== metricName || point.buckets == null) {
|
||||
continue;
|
||||
}
|
||||
for (const bucket of point.buckets) {
|
||||
const attributes: Attributes = {
|
||||
...point.attributes,
|
||||
le: bucket.le,
|
||||
};
|
||||
result.observe(bucket.cumulativeCount, attributes);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const fieldCallback =
|
||||
(metricName: string, field: "count" | "sum") =>
|
||||
(result: ObservableResult) => {
|
||||
for (const point of snapshotLancedbMetrics()) {
|
||||
if (point.name !== metricName) {
|
||||
continue;
|
||||
}
|
||||
const value = point[field];
|
||||
if (value != null) {
|
||||
result.observe(value, point.attributes);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
for (const desc of lancedbMetricsCatalog()) {
|
||||
const unit = desc.unit ?? "";
|
||||
if (desc.kind === "counter") {
|
||||
const counter = meter.createObservableCounter(desc.name, {
|
||||
unit,
|
||||
description: desc.description,
|
||||
});
|
||||
counter.addCallback(scalarCallback(desc.name));
|
||||
} else if (desc.kind === "gauge") {
|
||||
const gauge = meter.createObservableGauge(desc.name, {
|
||||
unit,
|
||||
description: desc.description,
|
||||
});
|
||||
gauge.addCallback(scalarCallback(desc.name));
|
||||
} else if (desc.kind === "histogram") {
|
||||
// `_bucket` and `_count` observe cumulative sample counts, not the
|
||||
// histogram's measured quantity, so they are unitless; only `_sum`
|
||||
// carries the histogram's unit.
|
||||
const bucket = meter.createObservableCounter(`${desc.name}_bucket`, {
|
||||
description: `${desc.description} (cumulative buckets)`,
|
||||
});
|
||||
bucket.addCallback(bucketCallback(desc.name));
|
||||
|
||||
const count = meter.createObservableCounter(`${desc.name}_count`, {
|
||||
description: `${desc.description} (count)`,
|
||||
});
|
||||
count.addCallback(fieldCallback(desc.name, "count"));
|
||||
|
||||
const sum = meter.createObservableCounter(`${desc.name}_sum`, {
|
||||
unit,
|
||||
description: `${desc.description} (sum)`,
|
||||
});
|
||||
sum.addCallback(fieldCallback(desc.name, "sum"));
|
||||
}
|
||||
}
|
||||
|
||||
instrumented = true;
|
||||
return true;
|
||||
}
|
||||
+3
-15
@@ -79,8 +79,6 @@ export interface QueryExecutionOptions {
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
export type AnalyzePlanDistributedMetrics = "aggregate" | "per_worker" | "full";
|
||||
|
||||
export interface ColumnOrdering {
|
||||
columnName: string;
|
||||
ascending?: boolean;
|
||||
@@ -313,20 +311,13 @@ export class QueryBase<
|
||||
* KNNVectorDistance: metric=l2, metrics=[output_rows=1, elapsed_compute=114.333µs, output_batches=1]
|
||||
* LanceScan: uri=/path/to/data, projection=[vector], row_id=true, row_addr=false, ordered=false, metrics=[output_rows=1, elapsed_compute=103.626µs, bytes_read=549, iops=2, requests=2]
|
||||
*
|
||||
* @param distributedMetrics - How distributed worker metrics are displayed for remote query plans.
|
||||
* Defaults to `"aggregate"`.
|
||||
* @returns A query execution plan with runtime metrics for each step.
|
||||
*/
|
||||
async analyzePlan(
|
||||
distributedMetrics?: AnalyzePlanDistributedMetrics,
|
||||
): Promise<string> {
|
||||
const distributedMetricsMode = distributedMetrics ?? "aggregate";
|
||||
async analyzePlan(): Promise<string> {
|
||||
if (this.inner instanceof Promise) {
|
||||
return this.inner.then((inner) =>
|
||||
inner.analyzePlan(distributedMetricsMode),
|
||||
);
|
||||
return this.inner.then((inner) => inner.analyzePlan());
|
||||
} else {
|
||||
return this.inner.analyzePlan(distributedMetricsMode);
|
||||
return this.inner.analyzePlan();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -371,9 +362,6 @@ export class StandardQueryBase<
|
||||
*
|
||||
* Filtering performance can often be improved by creating a scalar index
|
||||
* on the filter column(s).
|
||||
*
|
||||
* Calling this multiple times combines the filters with a logical AND rather
|
||||
* than replacing the previous filter.
|
||||
*/
|
||||
where(predicate: string): this {
|
||||
this.doCall((inner: NativeQueryType) => inner.onlyIf(predicate));
|
||||
|
||||
@@ -288,11 +288,12 @@ export function sanitizeMap(typeLike: object) {
|
||||
if (!("keysSorted" in typeLike) || typeof typeLike.keysSorted !== "boolean") {
|
||||
throw Error("Expected a Map type to have a `keysSorted` property");
|
||||
}
|
||||
if (typeLike.children.length !== 1) {
|
||||
throw Error("Expected a Map type to have exactly one child");
|
||||
}
|
||||
|
||||
return new Map_(sanitizeField(typeLike.children[0]), typeLike.keysSorted);
|
||||
return new Map_(
|
||||
// biome-ignore lint/suspicious/noExplicitAny: skip
|
||||
typeLike.children.map((field) => sanitizeField(field)) as any,
|
||||
typeLike.keysSorted,
|
||||
);
|
||||
}
|
||||
|
||||
export function sanitizeDuration(typeLike: object) {
|
||||
|
||||
@@ -25,12 +25,10 @@ import {
|
||||
AddColumnsSql,
|
||||
AddResult,
|
||||
AlterColumnsResult,
|
||||
BranchContents,
|
||||
DeleteResult,
|
||||
DropColumnsResult,
|
||||
IndexConfig,
|
||||
IndexStatistics,
|
||||
Branches as NativeBranches,
|
||||
OptimizeStats,
|
||||
TableStatistics,
|
||||
Tags,
|
||||
@@ -158,26 +156,6 @@ export interface Version {
|
||||
metadata: Record<string, string>;
|
||||
}
|
||||
|
||||
/** Token produced by the tokenizer configured on a full-text search index. */
|
||||
export interface FtsToken {
|
||||
/** Token text after tokenizer filters have been applied. */
|
||||
text: string;
|
||||
/** Token position used by full-text query matching. */
|
||||
position: number;
|
||||
}
|
||||
|
||||
export type TokenizeTableOptions =
|
||||
| {
|
||||
/** FTS-indexed column whose tokenizer should be used. */
|
||||
column: string;
|
||||
indexName?: never;
|
||||
}
|
||||
| {
|
||||
/** Name of the FTS index whose tokenizer should be used. */
|
||||
indexName: string;
|
||||
column?: never;
|
||||
};
|
||||
|
||||
/**
|
||||
* Specification selecting Lance's MemWAL LSM-style write path for
|
||||
* `mergeInsert`.
|
||||
@@ -605,17 +583,6 @@ export abstract class Table {
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
abstract unsetLsmWriteSpec(): Promise<void>;
|
||||
/**
|
||||
* Read the {@link LsmWriteSpec} currently installed on this table.
|
||||
*
|
||||
* Resolves to `undefined` when the MemWAL LSM write path is not enabled (no
|
||||
* spec has been set, or it was removed with {@link Table#unsetLsmWriteSpec}).
|
||||
* The returned spec — including its `maintainedIndexes` and
|
||||
* `writerConfigDefaults` — mirrors what was passed to
|
||||
* {@link Table#setLsmWriteSpec}.
|
||||
* @returns {Promise<LsmWriteSpec | undefined>}
|
||||
*/
|
||||
abstract getLsmWriteSpec(): Promise<LsmWriteSpec | undefined>;
|
||||
/**
|
||||
* Drain and close any cached MemWAL shard writers held for this table.
|
||||
*
|
||||
@@ -686,22 +653,6 @@ export abstract class Table {
|
||||
*/
|
||||
abstract tags(): Promise<Tags>;
|
||||
|
||||
/**
|
||||
* Get the branch manager for this table.
|
||||
*
|
||||
* Branches are isolated, writable lines of history forked from another
|
||||
* branch (or version). Writes on a branch do not affect `main`.
|
||||
*/
|
||||
abstract branches(): Promise<Branches>;
|
||||
|
||||
/**
|
||||
* The branch this table handle is scoped to, or `null` for the main branch.
|
||||
*
|
||||
* A handle returned by {@link Branches.create} or {@link Branches.checkout}
|
||||
* reports the branch it targets; a handle opened normally reports `null`.
|
||||
*/
|
||||
abstract currentBranch(): string | null;
|
||||
|
||||
/**
|
||||
* Restore the table to the currently checked out version
|
||||
*
|
||||
@@ -736,19 +687,6 @@ export abstract class Table {
|
||||
abstract optimize(options?: Partial<OptimizeOptions>): Promise<OptimizeStats>;
|
||||
/** List all indices that have been created with {@link Table.createIndex} */
|
||||
abstract listIndices(): Promise<IndexConfig[]>;
|
||||
/**
|
||||
* Tokenize a full-text search query using the tokenizer configured on an FTS index.
|
||||
*
|
||||
* Specify exactly one of `column` or `indexName`.
|
||||
*
|
||||
* Model-backed tokenizers such as `jieba/*` and `lindera/*` are rebuilt in
|
||||
* the client process from index metadata. For remote tables, this means the
|
||||
* same tokenizer model files must also exist locally.
|
||||
*/
|
||||
abstract tokenize(
|
||||
query: string,
|
||||
options: TokenizeTableOptions,
|
||||
): Promise<FtsToken[]>;
|
||||
/** Return the table as an arrow table */
|
||||
abstract toArrow(): Promise<ArrowTable>;
|
||||
|
||||
@@ -1135,15 +1073,6 @@ export class LocalTable extends Table {
|
||||
return await this.inner.unsetLsmWriteSpec();
|
||||
}
|
||||
|
||||
async getLsmWriteSpec(): Promise<LsmWriteSpec | undefined> {
|
||||
// The native binding types `specType` as a plain `string`; narrow it back
|
||||
// to the public union. The Rust `From` impl only ever emits one of the
|
||||
// three valid values, so the cast is safe.
|
||||
return ((await this.inner.getLsmWriteSpec()) ?? undefined) as
|
||||
| LsmWriteSpec
|
||||
| undefined;
|
||||
}
|
||||
|
||||
async closeLsmWriters(): Promise<void> {
|
||||
return await this.inner.closeLsmWriters();
|
||||
}
|
||||
@@ -1179,14 +1108,6 @@ export class LocalTable extends Table {
|
||||
return await this.inner.tags();
|
||||
}
|
||||
|
||||
async branches(): Promise<Branches> {
|
||||
return new Branches(await this.inner.branches());
|
||||
}
|
||||
|
||||
currentBranch(): string | null {
|
||||
return this.inner.currentBranch() ?? null;
|
||||
}
|
||||
|
||||
async optimize(options?: Partial<OptimizeOptions>): Promise<OptimizeStats> {
|
||||
let cleanupOlderThanMs;
|
||||
if (
|
||||
@@ -1206,17 +1127,6 @@ export class LocalTable extends Table {
|
||||
return await this.inner.listIndices();
|
||||
}
|
||||
|
||||
async tokenize(
|
||||
query: string,
|
||||
options: TokenizeTableOptions,
|
||||
): Promise<FtsToken[]> {
|
||||
return await this.inner.tokenize(
|
||||
query,
|
||||
options?.column,
|
||||
options?.indexName,
|
||||
);
|
||||
}
|
||||
|
||||
async toArrow(): Promise<ArrowTable> {
|
||||
return await this.query().toArrow();
|
||||
}
|
||||
@@ -1328,151 +1238,3 @@ export interface FieldMetadataUpdate {
|
||||
/** If true, replace the field's entire metadata map instead of merging. */
|
||||
replace?: boolean;
|
||||
}
|
||||
|
||||
/** Summary of a column in a branch diff. */
|
||||
export interface BranchColumnSummary {
|
||||
name: string;
|
||||
dataType: string;
|
||||
nullable: boolean;
|
||||
}
|
||||
|
||||
/** A column whose definition differs between main and the branch. */
|
||||
export interface BranchColumnChange {
|
||||
name: string;
|
||||
main: BranchColumnSummary;
|
||||
branch: BranchColumnSummary;
|
||||
}
|
||||
|
||||
/** Summary of an index in a branch diff. */
|
||||
export interface BranchIndexSummary {
|
||||
indexName: string;
|
||||
columns: string[];
|
||||
indexType?: string;
|
||||
status: string;
|
||||
}
|
||||
|
||||
/** Row-level comparison between main and the branch. */
|
||||
export interface BranchRowCountSummary {
|
||||
unchanged: number;
|
||||
newOnBase: number;
|
||||
newOnBranch: number;
|
||||
staleRecompute: number;
|
||||
inputsChanged: number;
|
||||
deltaAvailable: boolean;
|
||||
}
|
||||
|
||||
/** A reason why a branch cannot currently be merged. */
|
||||
export interface MergeBlocker {
|
||||
code: string;
|
||||
message: string;
|
||||
}
|
||||
|
||||
/** Read-only comparison of a branch against main. */
|
||||
export interface BranchDiff {
|
||||
fromBranch: string;
|
||||
parentVersion: number;
|
||||
mainVersion: number;
|
||||
branchVersion: number;
|
||||
baseMoved: boolean;
|
||||
rowCountMain: number;
|
||||
rowCountBranch: number;
|
||||
rowSummary: BranchRowCountSummary;
|
||||
addedColumns: BranchColumnSummary[];
|
||||
removedColumns: BranchColumnSummary[];
|
||||
changedColumns: BranchColumnChange[];
|
||||
addedIndexes: BranchIndexSummary[];
|
||||
removedIndexes: BranchIndexSummary[];
|
||||
mergeable: boolean;
|
||||
mergeBlockers: MergeBlocker[];
|
||||
}
|
||||
|
||||
/** Changes that would be, or were, promoted by a branch merge. */
|
||||
export interface MergePreview {
|
||||
promotedColumns: string[];
|
||||
}
|
||||
|
||||
/** Result of previewing or attempting a branch merge. */
|
||||
export interface MergeBranchResult {
|
||||
status: "ready" | "rejected" | "notImplemented" | "merged" | "unknown";
|
||||
diff: BranchDiff;
|
||||
preview: MergePreview;
|
||||
mainVersionAfter?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Branch manager for a {@link Table}.
|
||||
*
|
||||
* Unlike tags, `create` and `checkout` return a new {@link Table} handle scoped
|
||||
* to the branch; writes on it do not affect `main`.
|
||||
*/
|
||||
export class Branches {
|
||||
#inner: NativeBranches;
|
||||
|
||||
/**
|
||||
* Construct a Branches manager. Internal use only.
|
||||
* @hidden
|
||||
*/
|
||||
constructor(inner: NativeBranches) {
|
||||
this.#inner = inner;
|
||||
}
|
||||
|
||||
/** List all branches, mapping name to branch metadata. */
|
||||
async list(): Promise<Record<string, BranchContents>> {
|
||||
return await this.#inner.list();
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a branch and return a handle scoped to it.
|
||||
*
|
||||
* @param name Name of the new branch.
|
||||
* @param fromRef Source branch to fork from. Defaults to `main`.
|
||||
* @param fromVersion A specific version on `fromRef`. Defaults to latest.
|
||||
*/
|
||||
async create(
|
||||
name: string,
|
||||
fromRef?: string,
|
||||
fromVersion?: number,
|
||||
): Promise<Table> {
|
||||
return new LocalTable(await this.#inner.create(name, fromRef, fromVersion));
|
||||
}
|
||||
|
||||
/**
|
||||
* Check out an existing branch and return a handle scoped to it.
|
||||
*
|
||||
* With `version` set, the returned handle is pinned to that version of the
|
||||
* branch (a read-only, detached view); otherwise it tracks the branch's
|
||||
* latest and stays writable.
|
||||
*/
|
||||
async checkout(name: string, version?: number): Promise<Table> {
|
||||
return new LocalTable(await this.#inner.checkout(name, version));
|
||||
}
|
||||
|
||||
/** Delete a branch. */
|
||||
async delete(name: string): Promise<void> {
|
||||
return await this.#inner.delete(name);
|
||||
}
|
||||
|
||||
/** Compare a branch against main without modifying either branch. */
|
||||
async diff(fromBranch: string): Promise<BranchDiff> {
|
||||
return (await this.#inner.diff(fromBranch)) as unknown as BranchDiff;
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge a branch into main.
|
||||
*
|
||||
* Set `dryRun` to `true` to preview the merge. A rejected merge resolves
|
||||
* with `status: "rejected"` instead of throwing.
|
||||
*
|
||||
* @param fromBranch Branch to merge from.
|
||||
* @param dryRun When true, only preview the merge. Defaults to false.
|
||||
*/
|
||||
async merge(
|
||||
fromBranch: string,
|
||||
dryRun: boolean = false,
|
||||
): Promise<MergeBranchResult> {
|
||||
return (await this.#inner.merge(
|
||||
fromBranch,
|
||||
dryRun,
|
||||
)) as unknown as MergeBranchResult;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-darwin-arm64",
|
||||
"version": "0.33.0-beta.0",
|
||||
"version": "0.30.1-beta.2",
|
||||
"os": ["darwin"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.darwin-arm64.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-arm64-gnu",
|
||||
"version": "0.33.0-beta.0",
|
||||
"version": "0.30.1-beta.2",
|
||||
"os": ["linux"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.linux-arm64-gnu.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-arm64-musl",
|
||||
"version": "0.33.0-beta.0",
|
||||
"version": "0.30.1-beta.2",
|
||||
"os": ["linux"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.linux-arm64-musl.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-x64-gnu",
|
||||
"version": "0.33.0-beta.0",
|
||||
"version": "0.30.1-beta.2",
|
||||
"os": ["linux"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.linux-x64-gnu.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-x64-musl",
|
||||
"version": "0.33.0-beta.0",
|
||||
"version": "0.30.1-beta.2",
|
||||
"os": ["linux"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.linux-x64-musl.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-win32-arm64-msvc",
|
||||
"version": "0.33.0-beta.0",
|
||||
"version": "0.30.1-beta.2",
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-win32-x64-msvc",
|
||||
"version": "0.33.0-beta.0",
|
||||
"version": "0.30.1-beta.2",
|
||||
"os": ["win32"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.win32-x64-msvc.node",
|
||||
|
||||
Generated
+7
-78
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb",
|
||||
"version": "0.33.0-beta.0",
|
||||
"version": "0.30.1-beta.1",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "@lancedb/lancedb",
|
||||
"version": "0.33.0-beta.0",
|
||||
"version": "0.30.1-beta.1",
|
||||
"cpu": [
|
||||
"x64",
|
||||
"arm64"
|
||||
@@ -18,7 +18,6 @@
|
||||
"win32"
|
||||
],
|
||||
"dependencies": {
|
||||
"@opentelemetry/api": "^1.9.0",
|
||||
"reflect-metadata": "^0.2.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
@@ -27,8 +26,7 @@
|
||||
"@aws-sdk/client-s3": "3.1003.0",
|
||||
"@biomejs/biome": "^1.7.3",
|
||||
"@jest/globals": "^29.7.0",
|
||||
"@napi-rs/cli": "3.7.0",
|
||||
"@opentelemetry/sdk-metrics": "^1.30.0",
|
||||
"@napi-rs/cli": "3.5.1",
|
||||
"@types/axios": "^0.14.0",
|
||||
"@types/jest": "^29.1.2",
|
||||
"@types/node": "22.7.4",
|
||||
@@ -2944,9 +2942,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/cli": {
|
||||
"version": "3.7.0",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-3.7.0.tgz",
|
||||
"integrity": "sha512-3d3+rmxlOIV/G1zPWeX4PCxuYnhcCQM2BvY9rtimC8RO0dFR9gtYP+Grov+WoduZtfWRj5N1XvytWeRxxCk5zw==",
|
||||
"version": "3.5.1",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-3.5.1.tgz",
|
||||
"integrity": "sha512-XBfLQRDcB3qhu6bazdMJsecWW55kR85l5/k0af9BIBELXQSsCFU0fzug7PX8eQp6vVdm7W/U3z6uP5WmITB2Gw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -2956,7 +2954,7 @@
|
||||
"@octokit/rest": "^22.0.1",
|
||||
"clipanion": "^4.0.0-rc.4",
|
||||
"colorette": "^2.0.20",
|
||||
"emnapi": "^1.10.0",
|
||||
"emnapi": "^1.7.1",
|
||||
"es-toolkit": "^1.41.0",
|
||||
"js-yaml": "^4.1.0",
|
||||
"obug": "^2.0.0",
|
||||
@@ -4150,75 +4148,6 @@
|
||||
"@octokit/openapi-types": "^27.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/api": {
|
||||
"version": "1.9.1",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz",
|
||||
"integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=8.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/core": {
|
||||
"version": "1.30.1",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.30.1.tgz",
|
||||
"integrity": "sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@opentelemetry/semantic-conventions": "1.28.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@opentelemetry/api": ">=1.0.0 <1.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/resources": {
|
||||
"version": "1.30.1",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.30.1.tgz",
|
||||
"integrity": "sha512-5UxZqiAgLYGFjS4s9qm5mBVo433u+dSPUFWVWXmLAD4wB65oMCoXaJP1KJa9DIYYMeHu3z4BZcStG3LC593cWA==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@opentelemetry/core": "1.30.1",
|
||||
"@opentelemetry/semantic-conventions": "1.28.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@opentelemetry/api": ">=1.0.0 <1.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/sdk-metrics": {
|
||||
"version": "1.30.1",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-1.30.1.tgz",
|
||||
"integrity": "sha512-q9zcZ0Okl8jRgmy7eNW3Ku1XSgg3sDLa5evHZpCwjspw7E8Is4K/haRPDJrBcX3YSn/Y7gUvFnByNYEKQNbNog==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@opentelemetry/core": "1.30.1",
|
||||
"@opentelemetry/resources": "1.30.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@opentelemetry/api": ">=1.3.0 <1.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/semantic-conventions": {
|
||||
"version": "1.28.0",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.28.0.tgz",
|
||||
"integrity": "sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
}
|
||||
},
|
||||
"node_modules/@protobufjs/aspromise": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
|
||||
|
||||
+2
-4
@@ -11,7 +11,7 @@
|
||||
"ann"
|
||||
],
|
||||
"private": false,
|
||||
"version": "0.33.0-beta.0",
|
||||
"version": "0.30.1-beta.2",
|
||||
"main": "dist/index.js",
|
||||
"exports": {
|
||||
".": "./dist/index.js",
|
||||
@@ -43,8 +43,7 @@
|
||||
"@aws-sdk/client-s3": "3.1003.0",
|
||||
"@biomejs/biome": "^1.7.3",
|
||||
"@jest/globals": "^29.7.0",
|
||||
"@napi-rs/cli": "3.7.0",
|
||||
"@opentelemetry/sdk-metrics": "^1.30.0",
|
||||
"@napi-rs/cli": "3.5.1",
|
||||
"@types/axios": "^0.14.0",
|
||||
"@types/jest": "^29.1.2",
|
||||
"@types/node": "22.7.4",
|
||||
@@ -93,7 +92,6 @@
|
||||
"version": "napi version"
|
||||
},
|
||||
"dependencies": {
|
||||
"@opentelemetry/api": "^1.9.0",
|
||||
"reflect-metadata": "^0.2.2"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
|
||||
Generated
+5
-58
@@ -8,9 +8,6 @@ importers:
|
||||
|
||||
.:
|
||||
dependencies:
|
||||
'@opentelemetry/api':
|
||||
specifier: ^1.9.0
|
||||
version: 1.9.1
|
||||
apache-arrow:
|
||||
specifier: '>=15.0.0 <=18.1.0'
|
||||
version: 18.1.0
|
||||
@@ -34,11 +31,8 @@ importers:
|
||||
specifier: ^29.7.0
|
||||
version: 29.7.0
|
||||
'@napi-rs/cli':
|
||||
specifier: 3.7.0
|
||||
version: 3.7.0(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)(@types/node@22.7.4)
|
||||
'@opentelemetry/sdk-metrics':
|
||||
specifier: ^1.30.0
|
||||
version: 1.30.1(@opentelemetry/api@1.9.1)
|
||||
specifier: 3.5.1
|
||||
version: 3.5.1(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)(@types/node@22.7.4)
|
||||
'@types/axios':
|
||||
specifier: ^0.14.0
|
||||
version: 0.14.4
|
||||
@@ -893,8 +887,8 @@ packages:
|
||||
'@jridgewell/trace-mapping@0.3.31':
|
||||
resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==}
|
||||
|
||||
'@napi-rs/cli@3.7.0':
|
||||
resolution: {integrity: sha512-3d3+rmxlOIV/G1zPWeX4PCxuYnhcCQM2BvY9rtimC8RO0dFR9gtYP+Grov+WoduZtfWRj5N1XvytWeRxxCk5zw==}
|
||||
'@napi-rs/cli@3.5.1':
|
||||
resolution: {integrity: sha512-XBfLQRDcB3qhu6bazdMJsecWW55kR85l5/k0af9BIBELXQSsCFU0fzug7PX8eQp6vVdm7W/U3z6uP5WmITB2Gw==}
|
||||
engines: {node: '>= 16'}
|
||||
hasBin: true
|
||||
peerDependencies:
|
||||
@@ -1313,32 +1307,6 @@ packages:
|
||||
'@octokit/types@16.0.0':
|
||||
resolution: {integrity: sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==}
|
||||
|
||||
'@opentelemetry/api@1.9.1':
|
||||
resolution: {integrity: sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==}
|
||||
engines: {node: '>=8.0.0'}
|
||||
|
||||
'@opentelemetry/core@1.30.1':
|
||||
resolution: {integrity: sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ==}
|
||||
engines: {node: '>=14'}
|
||||
peerDependencies:
|
||||
'@opentelemetry/api': '>=1.0.0 <1.10.0'
|
||||
|
||||
'@opentelemetry/resources@1.30.1':
|
||||
resolution: {integrity: sha512-5UxZqiAgLYGFjS4s9qm5mBVo433u+dSPUFWVWXmLAD4wB65oMCoXaJP1KJa9DIYYMeHu3z4BZcStG3LC593cWA==}
|
||||
engines: {node: '>=14'}
|
||||
peerDependencies:
|
||||
'@opentelemetry/api': '>=1.0.0 <1.10.0'
|
||||
|
||||
'@opentelemetry/sdk-metrics@1.30.1':
|
||||
resolution: {integrity: sha512-q9zcZ0Okl8jRgmy7eNW3Ku1XSgg3sDLa5evHZpCwjspw7E8Is4K/haRPDJrBcX3YSn/Y7gUvFnByNYEKQNbNog==}
|
||||
engines: {node: '>=14'}
|
||||
peerDependencies:
|
||||
'@opentelemetry/api': '>=1.3.0 <1.10.0'
|
||||
|
||||
'@opentelemetry/semantic-conventions@1.28.0':
|
||||
resolution: {integrity: sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA==}
|
||||
engines: {node: '>=14'}
|
||||
|
||||
'@protobufjs/aspromise@1.1.2':
|
||||
resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==}
|
||||
|
||||
@@ -4614,7 +4582,7 @@ snapshots:
|
||||
'@jridgewell/resolve-uri': 3.1.2
|
||||
'@jridgewell/sourcemap-codec': 1.5.5
|
||||
|
||||
'@napi-rs/cli@3.7.0(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)(@types/node@22.7.4)':
|
||||
'@napi-rs/cli@3.5.1(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)(@types/node@22.7.4)':
|
||||
dependencies:
|
||||
'@inquirer/prompts': 8.4.3(@types/node@22.7.4)
|
||||
'@napi-rs/cross-toolchain': 1.0.3(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)
|
||||
@@ -4957,27 +4925,6 @@ snapshots:
|
||||
dependencies:
|
||||
'@octokit/openapi-types': 27.0.0
|
||||
|
||||
'@opentelemetry/api@1.9.1': {}
|
||||
|
||||
'@opentelemetry/core@1.30.1(@opentelemetry/api@1.9.1)':
|
||||
dependencies:
|
||||
'@opentelemetry/api': 1.9.1
|
||||
'@opentelemetry/semantic-conventions': 1.28.0
|
||||
|
||||
'@opentelemetry/resources@1.30.1(@opentelemetry/api@1.9.1)':
|
||||
dependencies:
|
||||
'@opentelemetry/api': 1.9.1
|
||||
'@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1)
|
||||
'@opentelemetry/semantic-conventions': 1.28.0
|
||||
|
||||
'@opentelemetry/sdk-metrics@1.30.1(@opentelemetry/api@1.9.1)':
|
||||
dependencies:
|
||||
'@opentelemetry/api': 1.9.1
|
||||
'@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1)
|
||||
'@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.1)
|
||||
|
||||
'@opentelemetry/semantic-conventions@1.28.0': {}
|
||||
|
||||
'@protobufjs/aspromise@1.1.2':
|
||||
optional: true
|
||||
|
||||
|
||||
@@ -112,12 +112,6 @@ impl Connection {
|
||||
|
||||
builder = builder.client_config(rust_config);
|
||||
|
||||
if let Some(oauth_config) = options.oauth_config {
|
||||
let config: lancedb::remote::oauth::OAuthConfig =
|
||||
oauth_config.try_into().default_error()?;
|
||||
builder = builder.oauth_config(config);
|
||||
}
|
||||
|
||||
if let Some(api_key) = options.api_key {
|
||||
builder = builder.api_key(&api_key);
|
||||
}
|
||||
|
||||
+1
-70
@@ -4,16 +4,13 @@
|
||||
use std::sync::Mutex;
|
||||
|
||||
use lancedb::index::Index as LanceDbIndex;
|
||||
use lancedb::index::scalar::{BTreeIndexBuilder, FmIndexBuilder, FtsIndexBuilder};
|
||||
use lancedb::index::scalar::{BTreeIndexBuilder, FtsIndexBuilder};
|
||||
use lancedb::index::vector::{
|
||||
IvfFlatIndexBuilder, IvfHnswPqIndexBuilder, IvfHnswSqIndexBuilder, IvfPqIndexBuilder,
|
||||
IvfRqIndexBuilder,
|
||||
};
|
||||
use lancedb::tokenize as lancedb_tokenize;
|
||||
use napi_derive::napi;
|
||||
|
||||
use crate::error::NapiErrorExt;
|
||||
use crate::table::FtsToken;
|
||||
use crate::util::parse_distance_type;
|
||||
|
||||
#[napi]
|
||||
@@ -33,65 +30,6 @@ impl Index {
|
||||
}
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
#[allow(dead_code, clippy::too_many_arguments)]
|
||||
pub fn tokenize(
|
||||
query: String,
|
||||
base_tokenizer: Option<String>,
|
||||
language: Option<String>,
|
||||
max_token_length: Option<u32>,
|
||||
lower_case: Option<bool>,
|
||||
stem: Option<bool>,
|
||||
remove_stop_words: Option<bool>,
|
||||
ascii_folding: Option<bool>,
|
||||
ngram_min_length: Option<u32>,
|
||||
ngram_max_length: Option<u32>,
|
||||
prefix_only: Option<bool>,
|
||||
) -> napi::Result<Vec<FtsToken>> {
|
||||
let mut opts = FtsIndexBuilder::default();
|
||||
if let Some(base_tokenizer) = base_tokenizer {
|
||||
opts = opts.base_tokenizer(base_tokenizer);
|
||||
}
|
||||
if let Some(language) = language {
|
||||
opts = opts.language(&language).map_err(|_| {
|
||||
napi::Error::from_reason(format!(
|
||||
"LanceDB does not support the requested language: '{}'",
|
||||
language
|
||||
))
|
||||
})?;
|
||||
}
|
||||
if let Some(max_token_length) = max_token_length {
|
||||
opts = opts.max_token_length(Some(max_token_length as usize));
|
||||
}
|
||||
if let Some(lower_case) = lower_case {
|
||||
opts = opts.lower_case(lower_case);
|
||||
}
|
||||
if let Some(stem) = stem {
|
||||
opts = opts.stem(stem);
|
||||
}
|
||||
if let Some(remove_stop_words) = remove_stop_words {
|
||||
opts = opts.remove_stop_words(remove_stop_words);
|
||||
}
|
||||
if let Some(ascii_folding) = ascii_folding {
|
||||
opts = opts.ascii_folding(ascii_folding);
|
||||
}
|
||||
if let Some(ngram_min_length) = ngram_min_length {
|
||||
opts = opts.ngram_min_length(ngram_min_length);
|
||||
}
|
||||
if let Some(ngram_max_length) = ngram_max_length {
|
||||
opts = opts.ngram_max_length(ngram_max_length);
|
||||
}
|
||||
if let Some(prefix_only) = prefix_only {
|
||||
opts = opts.ngram_prefix_only(prefix_only);
|
||||
}
|
||||
|
||||
Ok(lancedb_tokenize(&query, &opts)
|
||||
.default_error()?
|
||||
.into_iter()
|
||||
.map(FtsToken::from)
|
||||
.collect())
|
||||
}
|
||||
|
||||
#[napi]
|
||||
impl Index {
|
||||
#[napi(factory)]
|
||||
@@ -205,13 +143,6 @@ impl Index {
|
||||
}
|
||||
}
|
||||
|
||||
#[napi(factory)]
|
||||
pub fn fm() -> Self {
|
||||
Self {
|
||||
inner: Mutex::new(Some(LanceDbIndex::Fm(FmIndexBuilder::default()))),
|
||||
}
|
||||
}
|
||||
|
||||
#[napi(factory)]
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn fts(
|
||||
|
||||
@@ -12,7 +12,6 @@ mod header;
|
||||
mod index;
|
||||
mod iterator;
|
||||
pub mod merge;
|
||||
pub mod otel;
|
||||
pub mod permutation;
|
||||
mod query;
|
||||
pub mod remote;
|
||||
@@ -66,11 +65,6 @@ pub struct ConnectionOptions {
|
||||
/// (For LanceDB cloud only): the host to use for LanceDB cloud. Used
|
||||
/// for testing purposes.
|
||||
pub host_override: Option<String>,
|
||||
/// (For LanceDB cloud only): OAuth configuration for IdP-based
|
||||
/// authentication (e.g., Azure Entra ID). When set, token acquisition
|
||||
/// and refresh are handled entirely in Rust. TypeScript users should pass
|
||||
/// the public `OAuthConfig` type exported from `@lancedb/lancedb`.
|
||||
pub oauth_config: Option<remote::OAuthConfig>,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
|
||||
+6
-4
@@ -3,7 +3,7 @@
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use lancedb::{ipc::ipc_file_to_batches, table::merge::MergeInsertBuilder};
|
||||
use lancedb::{arrow::IntoArrow, ipc::ipc_file_to_batches, table::merge::MergeInsertBuilder};
|
||||
use napi::bindgen_prelude::*;
|
||||
use napi_derive::napi;
|
||||
|
||||
@@ -66,9 +66,11 @@ impl NativeMergeInsertBuilder {
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn execute(&self, buf: Buffer) -> napi::Result<MergeResult> {
|
||||
let data = ipc_file_to_batches(buf.to_vec()).map_err(|e| {
|
||||
napi::Error::from_reason(format!("Failed to read IPC file: {}", convert_error(&e)))
|
||||
})?;
|
||||
let data = ipc_file_to_batches(buf.to_vec())
|
||||
.and_then(IntoArrow::into_arrow)
|
||||
.map_err(|e| {
|
||||
napi::Error::from_reason(format!("Failed to read IPC file: {}", convert_error(&e)))
|
||||
})?;
|
||||
|
||||
let this = self.clone();
|
||||
|
||||
|
||||
@@ -1,119 +0,0 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
//! Node.js bindings over [`lancedb::metrics_otel`].
|
||||
//!
|
||||
//! The aggregation, catalog, and histogram bucketing all live in the LanceDB
|
||||
//! core crate; this module only converts the core snapshot types into napi
|
||||
//! objects and exposes the three entry points to JavaScript, where
|
||||
//! `lancedb/otel.ts` bridges them into the user's OpenTelemetry `MeterProvider`.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use lancedb::metrics_otel::{MetricPoint as CoreMetricPoint, MetricValue};
|
||||
use napi_derive::napi;
|
||||
|
||||
/// One cumulative histogram bucket: all samples with value `<= le`.
|
||||
#[napi(object)]
|
||||
pub struct MetricBucket {
|
||||
/// The inclusive upper bound of the bucket, or `"+Inf"` for the final bucket.
|
||||
pub le: String,
|
||||
/// Cumulative number of samples less than or equal to `le`.
|
||||
pub cumulative_count: f64,
|
||||
}
|
||||
|
||||
/// One aggregated metric data point. For counters and gauges only `value` is
|
||||
/// set; for histograms `buckets` (cumulative `le` counts), `count`, and `sum`
|
||||
/// are set.
|
||||
#[napi(object)]
|
||||
pub struct MetricPoint {
|
||||
pub name: String,
|
||||
pub kind: String,
|
||||
pub attributes: HashMap<String, String>,
|
||||
pub value: Option<f64>,
|
||||
pub buckets: Option<Vec<MetricBucket>>,
|
||||
pub count: Option<f64>,
|
||||
pub sum: Option<f64>,
|
||||
}
|
||||
|
||||
impl From<CoreMetricPoint> for MetricPoint {
|
||||
fn from(point: CoreMetricPoint) -> Self {
|
||||
let kind = point.kind.as_str().to_string();
|
||||
let (value, buckets, count, sum) = match point.value {
|
||||
MetricValue::Scalar(v) => (Some(v), None, None, None),
|
||||
MetricValue::Histogram {
|
||||
buckets,
|
||||
count,
|
||||
sum,
|
||||
} => (
|
||||
None,
|
||||
Some(
|
||||
buckets
|
||||
.into_iter()
|
||||
// Counts stay well within the f64-exact integer range
|
||||
// (2^53), so this cast is lossless in practice and keeps
|
||||
// the values plain JS numbers for OpenTelemetry.
|
||||
.map(|(le, cumulative_count)| MetricBucket {
|
||||
le,
|
||||
cumulative_count: cumulative_count as f64,
|
||||
})
|
||||
.collect(),
|
||||
),
|
||||
Some(count as f64),
|
||||
Some(sum),
|
||||
),
|
||||
};
|
||||
Self {
|
||||
name: point.name,
|
||||
kind,
|
||||
attributes: point.attributes,
|
||||
value,
|
||||
buckets,
|
||||
count,
|
||||
sum,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A described metric, used by the JavaScript layer to create instruments up front.
|
||||
#[napi(object)]
|
||||
pub struct MetricDescription {
|
||||
pub name: String,
|
||||
pub kind: String,
|
||||
pub unit: Option<String>,
|
||||
pub description: String,
|
||||
}
|
||||
|
||||
/// Install the LanceDB metrics recorder as the process-global `metrics` recorder.
|
||||
///
|
||||
/// Returns `true` if the recorder is installed (now or previously). Returns
|
||||
/// `false` if a *different* recorder is already installed — `metrics` allows
|
||||
/// only one global recorder per process, so LanceDB cannot coexist with another.
|
||||
#[napi]
|
||||
pub fn register_lancedb_metrics_recorder() -> bool {
|
||||
lancedb::metrics_otel::register_metrics_recorder()
|
||||
}
|
||||
|
||||
/// The catalog of described LanceDB metrics. Empty until the recorder is installed.
|
||||
#[napi]
|
||||
pub fn lancedb_metrics_catalog() -> Vec<MetricDescription> {
|
||||
lancedb::metrics_otel::metrics_catalog()
|
||||
.into_iter()
|
||||
.map(|desc| MetricDescription {
|
||||
name: desc.name,
|
||||
kind: desc.kind.as_str().to_string(),
|
||||
unit: desc.unit,
|
||||
description: desc.description,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// A point-in-time snapshot of every recorded metric. Empty until the recorder
|
||||
/// is installed.
|
||||
#[napi]
|
||||
pub fn snapshot_lancedb_metrics() -> Vec<MetricPoint> {
|
||||
lancedb::metrics_otel::snapshot_metrics()
|
||||
.into_iter()
|
||||
.map(MetricPoint::from)
|
||||
.collect()
|
||||
}
|
||||
@@ -16,7 +16,6 @@ pub struct SplitRandomOptions {
|
||||
pub counts: Option<Vec<i64>>,
|
||||
pub fixed: Option<i64>,
|
||||
pub seed: Option<i64>,
|
||||
pub clump_size: Option<i64>,
|
||||
pub split_names: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
@@ -126,15 +125,10 @@ impl PermutationBuilder {
|
||||
};
|
||||
|
||||
let seed = options.seed.map(|s| s as u64);
|
||||
let clump_size = options.clump_size.map(|c| c as u64);
|
||||
|
||||
self.modify(|builder| {
|
||||
builder.with_split_strategy(
|
||||
SplitStrategy::Random {
|
||||
seed,
|
||||
sizes,
|
||||
clump_size,
|
||||
},
|
||||
SplitStrategy::Random { seed, sizes },
|
||||
options.split_names.clone(),
|
||||
)
|
||||
})
|
||||
|
||||
+21
-56
@@ -19,7 +19,6 @@ use lancedb::index::scalar::{
|
||||
BooleanQuery, BoostQuery, FtsQuery, FullTextSearchQuery, MatchQuery, MultiMatchQuery, Occur,
|
||||
Operator, PhraseQuery,
|
||||
};
|
||||
use lancedb::query::AnalyzePlanDistributedMetrics;
|
||||
use lancedb::query::ExecutableQuery;
|
||||
use lancedb::query::Query as LanceDbQuery;
|
||||
use lancedb::query::QueryBase;
|
||||
@@ -48,28 +47,6 @@ impl From<ColumnOrdering> for LanceDbColumnOrdering {
|
||||
}
|
||||
}
|
||||
|
||||
fn analyze_plan_options(
|
||||
distributed_metrics: Option<String>,
|
||||
) -> napi::Result<QueryExecutionOptions> {
|
||||
let analyze_plan_distributed_metrics =
|
||||
match distributed_metrics.as_deref().unwrap_or("aggregate") {
|
||||
"aggregate" => AnalyzePlanDistributedMetrics::Aggregate,
|
||||
"per_worker" => AnalyzePlanDistributedMetrics::PerWorker,
|
||||
"full" => AnalyzePlanDistributedMetrics::Full,
|
||||
mode => {
|
||||
return Err(napi::Error::from_reason(format!(
|
||||
"Invalid distributedMetrics value '{}'. Expected one of: \
|
||||
'aggregate', 'per_worker', 'full'",
|
||||
mode
|
||||
)));
|
||||
}
|
||||
};
|
||||
|
||||
let mut options = QueryExecutionOptions::default();
|
||||
options.analyze_plan_distributed_metrics = analyze_plan_distributed_metrics;
|
||||
Ok(options)
|
||||
}
|
||||
|
||||
fn bytes_to_arrow_array(data: Uint8Array, dtype: String) -> napi::Result<Arc<dyn Array>> {
|
||||
let buf = arrow_buffer::Buffer::from(data.to_vec());
|
||||
let num_bytes = buf.len();
|
||||
@@ -223,17 +200,13 @@ impl Query {
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn analyze_plan(&self, distributed_metrics: Option<String>) -> napi::Result<String> {
|
||||
let options = analyze_plan_options(distributed_metrics)?;
|
||||
self.inner
|
||||
.analyze_plan_with_options(options)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
napi::Error::from_reason(format!(
|
||||
"Failed to execute analyze plan: {}",
|
||||
convert_error(&e)
|
||||
))
|
||||
})
|
||||
pub async fn analyze_plan(&self) -> napi::Result<String> {
|
||||
self.inner.analyze_plan().await.map_err(|e| {
|
||||
napi::Error::from_reason(format!(
|
||||
"Failed to execute analyze plan: {}",
|
||||
convert_error(&e)
|
||||
))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -439,17 +412,13 @@ impl VectorQuery {
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn analyze_plan(&self, distributed_metrics: Option<String>) -> napi::Result<String> {
|
||||
let options = analyze_plan_options(distributed_metrics)?;
|
||||
self.inner
|
||||
.analyze_plan_with_options(options)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
napi::Error::from_reason(format!(
|
||||
"Failed to execute analyze plan: {}",
|
||||
convert_error(&e)
|
||||
))
|
||||
})
|
||||
pub async fn analyze_plan(&self) -> napi::Result<String> {
|
||||
self.inner.analyze_plan().await.map_err(|e| {
|
||||
napi::Error::from_reason(format!(
|
||||
"Failed to execute analyze plan: {}",
|
||||
convert_error(&e)
|
||||
))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -522,17 +491,13 @@ impl TakeQuery {
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn analyze_plan(&self, distributed_metrics: Option<String>) -> napi::Result<String> {
|
||||
let options = analyze_plan_options(distributed_metrics)?;
|
||||
self.inner
|
||||
.analyze_plan_with_options(options)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
napi::Error::from_reason(format!(
|
||||
"Failed to execute analyze plan: {}",
|
||||
convert_error(&e)
|
||||
))
|
||||
})
|
||||
pub async fn analyze_plan(&self) -> napi::Result<String> {
|
||||
self.inner.analyze_plan().await.map_err(|e| {
|
||||
napi::Error::from_reason(format!(
|
||||
"Failed to execute analyze plan: {}",
|
||||
convert_error(&e)
|
||||
))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use lancedb::error::Error;
|
||||
use napi_derive::*;
|
||||
|
||||
/// Timeout configuration for remote HTTP client.
|
||||
@@ -141,84 +140,6 @@ impl From<TlsConfig> for lancedb::remote::TlsConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// OAuth configuration for LanceDB authentication.
|
||||
///
|
||||
/// This is the generated napi-rs binding shape. TypeScript users should prefer
|
||||
/// the public `OAuthConfig` type exported from `@lancedb/lancedb`.
|
||||
///
|
||||
/// All token acquisition and refresh is handled in the Rust layer.
|
||||
#[napi(object)]
|
||||
#[derive(Clone)]
|
||||
pub struct OAuthConfig {
|
||||
/// OIDC issuer URL or OAuth authority URL.
|
||||
/// For Azure: `https://login.microsoftonline.com/{tenant_id}/v2.0`
|
||||
pub issuer_url: String,
|
||||
/// Application / Client ID.
|
||||
pub client_id: String,
|
||||
/// OAuth scopes to request. For Azure managed identity, exactly one scope
|
||||
/// or resource is required. For example: `["api://{app_id}/.default"]`
|
||||
pub scopes: Vec<String>,
|
||||
/// Authentication flow: "client_credentials" or "azure_managed_identity"
|
||||
pub flow: Option<String>,
|
||||
/// Client secret (required for client_credentials).
|
||||
pub client_secret: Option<String>,
|
||||
/// Client ID for user-assigned managed identity (azure_managed_identity).
|
||||
pub managed_identity_client_id: Option<String>,
|
||||
/// Seconds before expiry to trigger proactive refresh (default: 300).
|
||||
/// Keep this well below the token TTL; if it is greater than or equal to
|
||||
/// the TTL, each request refreshes the token.
|
||||
pub refresh_buffer_secs: Option<u32>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for OAuthConfig {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("OAuthConfig")
|
||||
.field("issuer_url", &self.issuer_url)
|
||||
.field("client_id", &self.client_id)
|
||||
.field("scopes", &self.scopes)
|
||||
.field("flow", &self.flow)
|
||||
.field(
|
||||
"client_secret",
|
||||
&self.client_secret.as_deref().map(|_| "<redacted>"),
|
||||
)
|
||||
.field(
|
||||
"managed_identity_client_id",
|
||||
&self.managed_identity_client_id,
|
||||
)
|
||||
.field("refresh_buffer_secs", &self.refresh_buffer_secs)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<OAuthConfig> for lancedb::remote::oauth::OAuthConfig {
|
||||
type Error = Error;
|
||||
|
||||
fn try_from(config: OAuthConfig) -> Result<Self, Self::Error> {
|
||||
use lancedb::remote::oauth::OAuthFlow;
|
||||
|
||||
let flow = match config.flow.as_deref().unwrap_or("client_credentials") {
|
||||
"client_credentials" => OAuthFlow::ClientCredentials,
|
||||
"azure_managed_identity" => OAuthFlow::AzureManagedIdentity {
|
||||
client_id: config.managed_identity_client_id,
|
||||
},
|
||||
other => {
|
||||
return Err(Error::InvalidInput {
|
||||
message: format!("Unknown OAuth flow type: {other}"),
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
Ok(Self {
|
||||
issuer_url: config.issuer_url,
|
||||
client_id: config.client_id,
|
||||
client_secret: config.client_secret,
|
||||
scopes: config.scopes,
|
||||
flow,
|
||||
refresh_buffer_secs: config.refresh_buffer_secs.map(|v| v as u64),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl From<ClientConfig> for lancedb::remote::ClientConfig {
|
||||
fn from(config: ClientConfig) -> Self {
|
||||
Self {
|
||||
@@ -232,52 +153,6 @@ impl From<ClientConfig> for lancedb::remote::ClientConfig {
|
||||
tls_config: config.tls_config.map(Into::into),
|
||||
header_provider: None, // the header provider is set separately later
|
||||
user_id: config.user_id,
|
||||
// Resolved from LANCE_CLIENT_MAX_BYTES_PER_REQUEST or the default.
|
||||
max_bytes_per_request: None,
|
||||
// Resolved from LANCE_CLIENT_MAX_REQUEST_DURATION or the read timeout.
|
||||
max_request_duration: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_unknown_oauth_flow_returns_invalid_input() {
|
||||
let config = OAuthConfig {
|
||||
issuer_url: "https://issuer.example.com".to_string(),
|
||||
client_id: "client-id".to_string(),
|
||||
scopes: vec!["scope".to_string()],
|
||||
flow: Some("typo".to_string()),
|
||||
client_secret: None,
|
||||
managed_identity_client_id: None,
|
||||
refresh_buffer_secs: None,
|
||||
};
|
||||
|
||||
let err = lancedb::remote::oauth::OAuthConfig::try_from(config).unwrap_err();
|
||||
assert!(matches!(
|
||||
err,
|
||||
Error::InvalidInput { message }
|
||||
if message == "Unknown OAuth flow type: typo"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_oauth_config_debug_redacts_client_secret() {
|
||||
let config = OAuthConfig {
|
||||
issuer_url: "https://issuer.example.com".to_string(),
|
||||
client_id: "client-id".to_string(),
|
||||
scopes: vec!["scope".to_string()],
|
||||
flow: Some("client_credentials".to_string()),
|
||||
client_secret: Some("super-secret".to_string()),
|
||||
managed_identity_client_id: None,
|
||||
refresh_buffer_secs: None,
|
||||
};
|
||||
|
||||
let debug = format!("{config:?}");
|
||||
assert!(!debug.contains("super-secret"));
|
||||
assert!(debug.contains("client_secret: Some(\"<redacted>\")"));
|
||||
}
|
||||
}
|
||||
|
||||
+6
-258
@@ -3,13 +3,11 @@
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
|
||||
use lancedb::ipc::{ipc_file_to_batches, ipc_file_to_schema};
|
||||
use lancedb::table::{
|
||||
AddDataMode, ColumnAlteration as LanceColumnAlteration, Duration,
|
||||
FieldMetadataUpdate as LanceFieldMetadataUpdate, FtsToken as LanceDbFtsToken,
|
||||
NewColumnTransform, OptimizeAction, OptimizeOptions, Ref, Table as LanceDbTable,
|
||||
FieldMetadataUpdate as LanceFieldMetadataUpdate, NewColumnTransform, OptimizeAction,
|
||||
OptimizeOptions, Table as LanceDbTable,
|
||||
};
|
||||
use napi::bindgen_prelude::*;
|
||||
use napi::threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode};
|
||||
@@ -411,16 +409,6 @@ impl Table {
|
||||
.default_error()
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn get_lsm_write_spec(&self) -> napi::Result<Option<LsmWriteSpec>> {
|
||||
let spec = self
|
||||
.inner_ref()?
|
||||
.get_lsm_write_spec()
|
||||
.await
|
||||
.default_error()?;
|
||||
Ok(spec.map(LsmWriteSpec::from))
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn close_lsm_writers(&self) -> napi::Result<()> {
|
||||
self.inner_ref()?.close_lsm_writers().await.default_error()
|
||||
@@ -490,19 +478,6 @@ impl Table {
|
||||
})
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn branches(&self) -> napi::Result<Branches> {
|
||||
Ok(Branches {
|
||||
inner: self.inner_ref()?.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
/// The branch this handle is scoped to, or `null` for the main branch.
|
||||
#[napi]
|
||||
pub fn current_branch(&self) -> napi::Result<Option<String>> {
|
||||
Ok(self.inner_ref()?.current_branch())
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn optimize(
|
||||
&self,
|
||||
@@ -574,27 +549,6 @@ impl Table {
|
||||
.collect::<Vec<_>>())
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn tokenize(
|
||||
&self,
|
||||
query: String,
|
||||
column: Option<String>,
|
||||
index_name: Option<String>,
|
||||
) -> napi::Result<Vec<FtsToken>> {
|
||||
let table = self.inner_ref()?;
|
||||
let tokens = match (column.as_deref(), index_name.as_deref()) {
|
||||
(Some(_), Some(_)) | (None, None) => {
|
||||
return Err(napi::Error::from_reason(
|
||||
"Specify exactly one of 'column' or 'indexName'",
|
||||
));
|
||||
}
|
||||
(Some(column), None) => table.tokenize_with_column(&query, column).await,
|
||||
(None, Some(index_name)) => table.tokenize(&query, index_name).await,
|
||||
}
|
||||
.default_error()?;
|
||||
Ok(tokens.into_iter().map(FtsToken::from).collect())
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn index_stats(&self, index_name: String) -> napi::Result<Option<IndexStatistics>> {
|
||||
let tbl = self.inner_ref()?;
|
||||
@@ -641,43 +595,6 @@ pub struct IndexConfig {
|
||||
/// Currently this is always an array of size 1. In the future there may
|
||||
/// be more columns to represent composite indices.
|
||||
pub columns: Vec<String>,
|
||||
/// The UUID of the first segment of the index.
|
||||
///
|
||||
/// `undefined` for remote tables, which do not yet surface this.
|
||||
pub index_uuid: Option<String>,
|
||||
/// The protobuf type URL, a precise type identifier for the index.
|
||||
///
|
||||
/// `undefined` for remote tables.
|
||||
pub type_url: Option<String>,
|
||||
/// When the index was created.
|
||||
///
|
||||
/// `undefined` for remote tables or indices created before timestamps were tracked.
|
||||
pub created_at: Option<DateTime<Utc>>,
|
||||
/// The number of rows indexed, across all segments.
|
||||
///
|
||||
/// `undefined` for remote tables.
|
||||
pub num_indexed_rows: Option<i64>,
|
||||
/// The number of rows not yet covered by this index.
|
||||
///
|
||||
/// `undefined` for remote tables.
|
||||
pub num_unindexed_rows: Option<i64>,
|
||||
/// The total size in bytes of all index files across all segments.
|
||||
///
|
||||
/// `undefined` for remote tables or indices without size tracking.
|
||||
pub size_bytes: Option<i64>,
|
||||
/// The number of segments that make up the index.
|
||||
///
|
||||
/// `undefined` for remote tables.
|
||||
pub num_segments: Option<i32>,
|
||||
/// The on-disk index format version.
|
||||
///
|
||||
/// `undefined` for remote tables.
|
||||
pub index_version: Option<i32>,
|
||||
/// Index-type-specific details parsed as a JavaScript object.
|
||||
///
|
||||
/// Falls back to a raw string if JSON parsing fails. `undefined` for
|
||||
/// remote tables or when details are unavailable.
|
||||
pub index_details: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
impl From<lancedb::index::IndexConfig> for IndexConfig {
|
||||
@@ -687,35 +604,6 @@ impl From<lancedb::index::IndexConfig> for IndexConfig {
|
||||
index_type,
|
||||
columns: value.columns,
|
||||
name: value.name,
|
||||
index_uuid: value.index_uuid,
|
||||
type_url: value.type_url,
|
||||
created_at: value.created_at,
|
||||
num_indexed_rows: value.num_indexed_rows.map(|n| n as i64),
|
||||
num_unindexed_rows: value.num_unindexed_rows.map(|n| n as i64),
|
||||
size_bytes: value.size_bytes.map(|n| n as i64),
|
||||
num_segments: value.num_segments.map(|n| n as i32),
|
||||
index_version: value.index_version,
|
||||
index_details: value
|
||||
.index_details
|
||||
.and_then(|s| serde_json::from_str(&s).ok()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
/// A token produced by the tokenizer configured on a full-text search index.
|
||||
pub struct FtsToken {
|
||||
/// The token text after the index tokenizer has applied its filters.
|
||||
pub text: String,
|
||||
/// The token position used by full-text query matching.
|
||||
pub position: u32,
|
||||
}
|
||||
|
||||
impl From<LanceDbFtsToken> for FtsToken {
|
||||
fn from(token: LanceDbFtsToken) -> Self {
|
||||
Self {
|
||||
text: token.text,
|
||||
position: token.position,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -777,47 +665,6 @@ impl TryFrom<LsmWriteSpec> for lancedb::table::LsmWriteSpec {
|
||||
}
|
||||
}
|
||||
|
||||
impl From<lancedb::table::LsmWriteSpec> for LsmWriteSpec {
|
||||
fn from(spec: lancedb::table::LsmWriteSpec) -> Self {
|
||||
use lancedb::table::LsmWriteSpec as Native;
|
||||
match spec {
|
||||
Native::Bucket {
|
||||
column,
|
||||
num_buckets,
|
||||
maintained_indexes,
|
||||
writer_config_defaults,
|
||||
} => Self {
|
||||
spec_type: "bucket".to_string(),
|
||||
column: Some(column),
|
||||
num_buckets: Some(num_buckets),
|
||||
maintained_indexes: Some(maintained_indexes),
|
||||
writer_config_defaults: Some(writer_config_defaults),
|
||||
},
|
||||
Native::Identity {
|
||||
column,
|
||||
maintained_indexes,
|
||||
writer_config_defaults,
|
||||
} => Self {
|
||||
spec_type: "identity".to_string(),
|
||||
column: Some(column),
|
||||
num_buckets: None,
|
||||
maintained_indexes: Some(maintained_indexes),
|
||||
writer_config_defaults: Some(writer_config_defaults),
|
||||
},
|
||||
Native::Unsharded {
|
||||
maintained_indexes,
|
||||
writer_config_defaults,
|
||||
} => Self {
|
||||
spec_type: "unsharded".to_string(),
|
||||
column: None,
|
||||
num_buckets: None,
|
||||
maintained_indexes: Some(maintained_indexes),
|
||||
writer_config_defaults: Some(writer_config_defaults),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Statistics about a compaction operation.
|
||||
#[napi(object)]
|
||||
#[derive(Clone, Debug)]
|
||||
@@ -991,6 +838,9 @@ pub struct IndexStatistics {
|
||||
pub distance_type: Option<String>,
|
||||
/// The number of parts this index is split into.
|
||||
pub num_indices: Option<u32>,
|
||||
/// The KMeans loss value of the index,
|
||||
/// it is only present for vector indices.
|
||||
pub loss: Option<f64>,
|
||||
}
|
||||
impl From<lancedb::index::IndexStatistics> for IndexStatistics {
|
||||
fn from(value: lancedb::index::IndexStatistics) -> Self {
|
||||
@@ -1000,6 +850,7 @@ impl From<lancedb::index::IndexStatistics> for IndexStatistics {
|
||||
index_type: value.index_type.to_string(),
|
||||
distance_type: value.distance_type.map(|d| d.to_string()),
|
||||
num_indices: value.num_indices,
|
||||
loss: value.loss,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1209,13 +1060,6 @@ pub struct TagContents {
|
||||
pub manifest_size: i64,
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub struct BranchContents {
|
||||
pub parent_branch: Option<String>,
|
||||
pub parent_version: i64,
|
||||
pub manifest_size: i64,
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub struct Tags {
|
||||
inner: LanceDbTable,
|
||||
@@ -1284,99 +1128,3 @@ impl Tags {
|
||||
.default_error()
|
||||
}
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub struct Branches {
|
||||
inner: LanceDbTable,
|
||||
}
|
||||
|
||||
#[napi]
|
||||
impl Branches {
|
||||
#[napi]
|
||||
pub async fn list(&self) -> napi::Result<HashMap<String, BranchContents>> {
|
||||
let branches = self.inner.list_branches().await.default_error()?;
|
||||
let result = branches
|
||||
.into_iter()
|
||||
.map(|(k, v)| {
|
||||
(
|
||||
k,
|
||||
BranchContents {
|
||||
parent_branch: v.parent_branch,
|
||||
parent_version: v.parent_version as i64,
|
||||
manifest_size: v.manifest_size as i64,
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn create(
|
||||
&self,
|
||||
name: String,
|
||||
from_ref: Option<String>,
|
||||
from_version: Option<i64>,
|
||||
) -> napi::Result<Table> {
|
||||
let from_ref = from_ref.filter(|b| b != "main");
|
||||
let from_version = from_version
|
||||
.map(|v| {
|
||||
u64::try_from(v).map_err(|_| {
|
||||
napi::Error::from_reason("from_version must be a non-negative integer")
|
||||
})
|
||||
})
|
||||
.transpose()?;
|
||||
let from = Ref::Version(from_ref, from_version);
|
||||
let table = self
|
||||
.inner
|
||||
.create_branch(&name, from)
|
||||
.await
|
||||
.default_error()?;
|
||||
Ok(Table::new(table))
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn checkout(&self, name: String, version: Option<i64>) -> napi::Result<Table> {
|
||||
let version = version
|
||||
.map(|v| {
|
||||
u64::try_from(v)
|
||||
.map_err(|_| napi::Error::from_reason("version must be a non-negative integer"))
|
||||
})
|
||||
.transpose()?;
|
||||
let table = self
|
||||
.inner
|
||||
.checkout_branch(&name, version)
|
||||
.await
|
||||
.default_error()?;
|
||||
Ok(Table::new(table))
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub async fn delete(&self, name: String) -> napi::Result<()> {
|
||||
self.inner.delete_branch(&name).await.default_error()
|
||||
}
|
||||
|
||||
#[napi(ts_return_type = "Promise<Record<string, unknown>>")]
|
||||
pub async fn diff(&self, from_branch: String) -> napi::Result<serde_json::Value> {
|
||||
let diff = self.inner.diff_branch(&from_branch).await.default_error()?;
|
||||
serde_json::to_value(diff).map_err(|err| {
|
||||
napi::Error::from_reason(format!("failed to serialize branch diff: {err}"))
|
||||
})
|
||||
}
|
||||
|
||||
#[napi(ts_return_type = "Promise<Record<string, unknown>>")]
|
||||
pub async fn merge(
|
||||
&self,
|
||||
from_branch: String,
|
||||
dry_run: Option<bool>,
|
||||
) -> napi::Result<serde_json::Value> {
|
||||
let result = self
|
||||
.inner
|
||||
.merge_branch(&from_branch, dry_run.unwrap_or(false))
|
||||
.await
|
||||
.default_error()?;
|
||||
serde_json::to_value(result).map_err(|err| {
|
||||
napi::Error::from_reason(format!("failed to serialize branch merge result: {err}"))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,21 +0,0 @@
|
||||
{
|
||||
"name": "lancedb",
|
||||
"description": "Write, review, debug, and document LanceDB pipelines in Python and TypeScript that work across local LanceDB OSS tables and remote LanceDB Enterprise/Cloud tables, with idiomatic query/search patterns and performance defaults for ingestion, indexing, filtering, and diagnostics.",
|
||||
"version": "0.1.0",
|
||||
"author": {
|
||||
"name": "LanceDB"
|
||||
},
|
||||
"homepage": "https://www.lancedb.com",
|
||||
"keywords": [
|
||||
"lancedb",
|
||||
"vector-search",
|
||||
"full-text-search",
|
||||
"hybrid-search",
|
||||
"python",
|
||||
"typescript",
|
||||
"pipelines",
|
||||
"ingestion",
|
||||
"indexing",
|
||||
"performance"
|
||||
]
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
{
|
||||
"name": "lancedb",
|
||||
"version": "0.1.0",
|
||||
"description": "Codex plugin for building LanceDB pipelines in Python and TypeScript.",
|
||||
"author": {
|
||||
"name": "LanceDB"
|
||||
},
|
||||
"keywords": [
|
||||
"lancedb",
|
||||
"vector-search",
|
||||
"full-text-search",
|
||||
"hybrid-search",
|
||||
"python",
|
||||
"typescript",
|
||||
"pipelines"
|
||||
],
|
||||
"skills": "./skills/",
|
||||
"interface": {
|
||||
"displayName": "LanceDB",
|
||||
"shortDescription": "Build LanceDB pipelines in Python and TypeScript.",
|
||||
"longDescription": "Write, review, debug, and document LanceDB pipelines in Python and TypeScript that work across local LanceDB OSS tables and remote LanceDB Enterprise/Cloud tables, with idiomatic query/search patterns and performance defaults for ingestion, indexing, filtering, and diagnostics.",
|
||||
"developerName": "LanceDB",
|
||||
"websiteURL": "https://www.lancedb.com",
|
||||
"category": "Developer Tools",
|
||||
"capabilities": [
|
||||
"Developer Tools"
|
||||
],
|
||||
"defaultPrompt": "Create a LanceDB table, embed sample text, and run a vector search.",
|
||||
"composerIcon": "./assets/logo.png",
|
||||
"logo": "./assets/logo.png",
|
||||
"logoDark": "./assets/logo-dark.png"
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 10 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 35 KiB |
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user