Compare commits

..

2 Commits

Author SHA1 Message Date
Lu Qiu a65db7038e build: point lance dependencies at the sibling submodule checkout
Path deps replace the git+tag pins so the sophon build resolves lance
from src/lance directly, independent of the pinned tag's version.
2026-07-24 14:38:19 -07:00
lancedb automation ea133edfca chore: update lance dependency to v10.0.0-beta.2 2026-07-23 21:25:42 +00:00
150 changed files with 2471 additions and 12516 deletions
+1 -8
View File
@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.37.1-beta.0"
current_version = "0.32.0-beta.2"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.
@@ -75,13 +75,6 @@ filename = "nodejs/Cargo.toml"
replace = "\nversion = \"{new_version}\""
search = "\nversion = \"{current_version}\""
# The Python package takes its version from here (pyproject.toml declares
# `dynamic = ["version"]`, so maturin reads it out of the crate manifest).
[[tool.bumpversion.files]]
filename = "python/Cargo.toml"
replace = "\nversion = \"{new_version}\""
search = "\nversion = \"{current_version}\""
# Java documentation
[[tool.bumpversion.files]]
filename = "docs/src/java/java.md"
@@ -27,31 +27,19 @@ runs:
# Extract failed job names
FAILED_JOBS=$(echo "$JOB_RESULTS" | jq -r 'to_entries | map(select(.value.result == "failure")) | map(.key) | join(", ")')
TITLE="$WORKFLOW_NAME Failed ($FAILED_JOBS)"
# This action now also runs on nightly schedules, so a breakage that
# persists for a few days would otherwise file one issue per night.
# Comment on the open report instead when one already exists.
EXISTING=$(gh issue list --state open --label ci --limit 100 --json number,title \
| jq -r --arg title "$TITLE" 'map(select(.title == $title)) | .[0].number // empty')
if [ -n "$EXISTING" ]; then
gh issue comment "$EXISTING" --body "Failed again: $RUN_URL"
echo "Commented on existing issue #$EXISTING"
else
gh issue create \
--title "$TITLE" \
--body "The workflow **$WORKFLOW_NAME** failed during execution.
# Create issue with workflow name, failed jobs, and run URL
gh issue create \
--title "$WORKFLOW_NAME Failed ($FAILED_JOBS)" \
--body "The workflow **$WORKFLOW_NAME** failed during execution.
**Failed jobs:** $FAILED_JOBS
**Run URL:** $RUN_URL
Please investigate the failed jobs and address any issues." \
--label "ci"
--label "ci"
echo "Issue created successfully"
fi
echo "Issue created successfully"
else
echo "No job failures detected, skipping issue creation"
fi
+1
View File
@@ -6,6 +6,7 @@ on:
# We don't publish pre-releases for Rust. Crates.io is just a source
# distribution, so we don't need to publish pre-releases.
- "v*-beta*"
- "*-v*" # for example, python-vX.Y.Z
env:
# This env var is used by Swatinem/rust-cache@v2 for the cache
-85
View File
@@ -1,85 +0,0 @@
name: GitHub Release
# All SDKs share one version, so a single `vX.Y.Z` tag produces a single GitHub
# release covering all of them. The per-package publish workflows (PyPI, NPM,
# Cargo, Maven) trigger off the same tag independently.
on:
push:
tags:
- "v*"
permissions:
contents: read
jobs:
gh-release:
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
lfs: true
- name: Extract version
id: extract_version
env:
GITHUB_REF: ${{ github.ref }}
run: |
set -e
echo "Extracting tag and version from $GITHUB_REF"
if [[ $GITHUB_REF =~ refs/tags/v(.*) ]]; then
VERSION=${BASH_REMATCH[1]}
TAG=v$VERSION
echo "tag=$TAG" >> $GITHUB_OUTPUT
echo "version=$VERSION" >> $GITHUB_OUTPUT
else
echo "Failed to extract version from $GITHUB_REF"
exit 1
fi
echo "Extracted version $VERSION from $GITHUB_REF"
if [[ $VERSION =~ beta ]]; then
echo "This is a beta release"
echo "prerelease=true" >> $GITHUB_OUTPUT
# Get last release (that is not this one)
FROM_TAG=$(git tag --sort='version:refname' \
| grep ^v \
| grep -vF "$TAG" \
| python ci/semver_sort.py v \
| tail -n 1)
else
echo "This is a stable release"
echo "prerelease=false" >> $GITHUB_OUTPUT
# Get last stable tag (ignore betas)
FROM_TAG=$(git tag --sort='version:refname' \
| grep ^v \
| grep -vF "$TAG" \
| grep -v beta \
| python ci/semver_sort.py v \
| tail -n 1)
fi
echo "Found from tag $FROM_TAG"
echo "from_tag=$FROM_TAG" >> $GITHUB_OUTPUT
- name: Create Release Notes
id: release_notes
uses: mikepenz/release-changelog-builder-action@v4
with:
configuration: .github/release_notes.json
toTag: ${{ steps.extract_version.outputs.tag }}
fromTag: ${{ steps.extract_version.outputs.from_tag }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Create GH release
uses: softprops/action-gh-release@v2
with:
# Marking betas as pre-releases keeps them from taking the "Latest"
# badge on the releases page.
prerelease: ${{ steps.extract_version.outputs.prerelease }}
make_latest: ${{ steps.extract_version.outputs.prerelease == 'false' }}
tag_name: ${{ steps.extract_version.outputs.tag }}
token: ${{ secrets.GITHUB_TOKEN }}
generate_release_notes: false
name: LanceDB v${{ steps.extract_version.outputs.version }}
body: ${{ steps.release_notes.outputs.changelog }}
+29 -7
View File
@@ -1,14 +1,13 @@
name: Create release commit
# This workflow increments the version, tags it, and pushes it. All SDKs share
# a single version, so one tag releases all of them.
# This workflow increments versions, tags the version, and pushes it.
# When a tag is pushed, another workflow is triggered that creates a GH release
# and uploads the binaries. This workflow is only for creating the tag.
# This script will enforce that a minor version is incremented if there are any
# breaking changes since the last minor increment. A breaking change in any SDK
# bumps the minor version for all of them. If you wish to bypass this check, you
# can manually increment the version and push the tag.
# breaking changes since the last minor increment. However, it isn't able to
# differentiate between breaking changes in Node versus Python. If you wish to
# bypass this check, you can manually increment the version and push the tag.
on:
workflow_dispatch:
inputs:
@@ -25,6 +24,16 @@ on:
options:
- preview
- stable
python:
description: 'Make a Python release'
required: true
default: true
type: boolean
other:
description: 'Make a Node/Rust/Java release'
required: true
default: true
type: boolean
bump-minor:
description: 'Bump minor version'
required: true
@@ -56,12 +65,25 @@ jobs:
run: |
git config user.name 'Lance Release'
git config user.email 'lance-dev@lancedb.com'
- name: Bump version
- name: Bump Python version
if: ${{ inputs.python }}
working-directory: python
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
# Need to get the commit before bumping the version, so we can
# determine if there are breaking changes in the next step as well.
echo "COMMIT_BEFORE_BUMP=$(git rev-parse HEAD)" >> $GITHUB_ENV
pip install bump-my-version PyGithub packaging
bash ../ci/bump_version.sh ${{ inputs.type }} ${{ inputs.bump-minor }} python-v
- name: Bump Node/Rust version
if: ${{ inputs.other }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
pip install bump-my-version PyGithub packaging
bash ci/bump_version.sh ${{ inputs.type }} ${{ inputs.bump-minor }}
bash ci/bump_version.sh ${{ inputs.type }} ${{ inputs.bump-minor }} v $COMMIT_BEFORE_BUMP
bash ci/update_lockfiles.sh --amend
- name: Push new version tag
if: ${{ !inputs.dry_run }}
-15
View File
@@ -61,11 +61,6 @@ jobs:
sudo apt update
sudo apt install -y protobuf-compiler libssl-dev
- uses: Swatinem/rust-cache@v2
with:
# Restore everywhere, but only save from main. Per-PR saves are
# unreadable outside their own branch anyway, since GitHub scopes
# caches to the creating ref.
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Format Rust
run: cargo fmt --all -- --check
- name: Lint Rust
@@ -108,11 +103,6 @@ jobs:
cache: 'pnpm'
cache-dependency-path: nodejs/pnpm-lock.yaml
- uses: Swatinem/rust-cache@v2
with:
# Restore everywhere, but only save from main. Per-PR saves are
# unreadable outside their own branch anyway, since GitHub scopes
# caches to the creating ref.
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Install dependencies
run: |
sudo apt update
@@ -192,11 +182,6 @@ jobs:
cache-dependency-path: nodejs/pnpm-lock.yaml
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
with:
# Restore everywhere, but only save from main. Per-PR saves are
# unreadable outside their own branch anyway, since GitHub scopes
# caches to the creating ref.
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Install dependencies
run: |
brew install protobuf
+89 -95
View File
@@ -10,16 +10,10 @@ permissions:
on:
push:
branches:
- main
tags:
- "v*"
# The cross-compiled targets (musl especially) break from toolchain and
# dependency changes that nothing else in CI catches, and discovering that
# mid-release is expensive. A nightly run keeps that signal while dropping
# the full 8-target release matrix from all ~90 pushes to main each month.
# `report-failure` files an issue when a nightly breaks.
schedule:
- cron: "0 8 * * *"
workflow_dispatch:
pull_request:
# This should trigger a dry run (we skip the final publish step)
paths:
@@ -32,6 +26,73 @@ concurrency:
cancel-in-progress: true
jobs:
gh-release:
if: startsWith(github.ref, 'refs/tags/v')
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
lfs: true
- name: Extract version
id: extract_version
env:
GITHUB_REF: ${{ github.ref }}
run: |
set -e
echo "Extracting tag and version from $GITHUB_REF"
if [[ $GITHUB_REF =~ refs/tags/v(.*) ]]; then
VERSION=${BASH_REMATCH[1]}
TAG=v$VERSION
echo "tag=$TAG" >> $GITHUB_OUTPUT
echo "version=$VERSION" >> $GITHUB_OUTPUT
else
echo "Failed to extract version from $GITHUB_REF"
exit 1
fi
echo "Extracted version $VERSION from $GITHUB_REF"
if [[ $VERSION =~ beta ]]; then
echo "This is a beta release"
# Get last release (that is not this one)
FROM_TAG=$(git tag --sort='version:refname' \
| grep ^v \
| grep -vF "$TAG" \
| python ci/semver_sort.py v \
| tail -n 1)
else
echo "This is a stable release"
# Get last stable tag (ignore betas)
FROM_TAG=$(git tag --sort='version:refname' \
| grep ^v \
| grep -vF "$TAG" \
| grep -v beta \
| python ci/semver_sort.py v \
| tail -n 1)
fi
echo "Found from tag $FROM_TAG"
echo "from_tag=$FROM_TAG" >> $GITHUB_OUTPUT
- name: Create Release Notes
id: release_notes
uses: mikepenz/release-changelog-builder-action@v4
with:
configuration: .github/release_notes.json
toTag: ${{ steps.extract_version.outputs.tag }}
fromTag: ${{ steps.extract_version.outputs.from_tag }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Create GH release
uses: softprops/action-gh-release@v2
with:
prerelease: ${{ contains('beta', github.ref) }}
tag_name: ${{ steps.extract_version.outputs.tag }}
token: ${{ secrets.GITHUB_TOKEN }}
generate_release_notes: false
name: Node/Rust LanceDB v${{ steps.extract_version.outputs.version }}
body: ${{ steps.release_notes.outputs.changelog }}
build-lancedb:
strategy:
fail-fast: false
@@ -40,18 +101,9 @@ jobs:
- target: aarch64-apple-darwin
host: macos-latest
features: fp16kernels
pre_build: |-
brew install protobuf
# Fat LTO (the workspace default in .cargo/config.toml) is
# single-threaded and is the peak-memory step of the build. On
# this runner it accounted for ~111 of the job's ~113 minutes,
# making it the critical path of the entire publish pipeline.
# ThinLTO parallelizes it across the runner's cores, for a few
# percent of runtime performance.
export CARGO_PROFILE_RELEASE_LTO=thin
export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
pre_build: brew install protobuf
- target: x86_64-pc-windows-msvc
host: windows-2025
host: windows-2025-8x-x64
features: ","
pre_build: |-
choco install --no-progress protoc ninja nasm
@@ -59,19 +111,19 @@ jobs:
# There is an issue where choco doesn't add nasm to the path
export PATH="$PATH:/c/Program Files/NASM"
nasm -v
# See the ThinLTO note on aarch64-apple-darwin above. Keeping
# peak memory down is also what lets this run on the standard
# 4-core runner: the 8-core larger runner was only needed to
# stop fat LTO from OOMing rustc-LLVM.
# Fat LTO of the cdylib is single-threaded and the peak-memory
# step of the build, and had started hitting rustc-LLVM OOM on the
# Windows runners. ThinLTO parallelizes it across the runner's
# cores and keeps peak memory well under the limit.
export CARGO_PROFILE_RELEASE_LTO=thin
export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
- target: aarch64-pc-windows-msvc
host: windows-2025
host: windows-2025-8x-x64
features: ","
pre_build: |-
choco install --no-progress protoc
rustup target add aarch64-pc-windows-msvc
# See the ThinLTO note on aarch64-apple-darwin above.
# See ThinLTO note on the x86_64-pc-windows-msvc target above.
export CARGO_PROFILE_RELEASE_LTO=thin
export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
- target: x86_64-unknown-linux-gnu
@@ -146,49 +198,16 @@ jobs:
with:
toolchain: stable
targets: ${{ matrix.settings.target }}
# These builds were entirely uncached: the old key was static, so
# `actions/cache` (which only writes on a miss) could never refresh it,
# and the multi-GB whole-`target/` copy it tried to store never fit the
# repo's cache budget, so no entry was ever saved. rust-cache prunes
# `target/` to dependency artifacts and keys on Cargo.lock plus the rustc
# version, which both fixes the key and keeps entries a sane size.
#
# This caches dependency *compilation* only. The LTO link of the cdylib
# re-runs regardless, since the local crate changes every time, so the
# win is larger on the non-LTO jobs than here.
- name: Cache cargo (native builds)
uses: Swatinem/rust-cache@v2
if: ${{ !matrix.settings.docker }}
- name: Cache cargo
uses: actions/cache@v5
with:
# The release profile and per-target dirs differ from what the test
# workflows cache, so these need to be separate entries.
key: release-${{ matrix.settings.target }}
# Only the nightly run on main writes, so tag and PR runs restore a
# warm entry without every dependabot PR writing its own (which would
# be unreadable elsewhere anyway, since GitHub scopes caches to the
# creating ref). The nightly cadence also keeps entries inside
# GitHub's 7-day eviction window, which a tag-only trigger would not.
save-if: ${{ github.ref == 'refs/heads/main' }}
# Docker builds can use rust-cache too. `target/` already lives on the
# host because the whole workspace is bind-mounted into the container, and
# rust-cache's prune and save run host-side, so they can manage it -- which
# is what keeps the entry to dependency artifacts rather than a multi-GB
# copy of everything.
#
# Two differences from the native builds. The container's CARGO_HOME is
# bind-mounted from `.cargo-cache` rather than the host's ~/.cargo, so that
# has to be cached explicitly. And the key is derived from the *host* rustc
# version, which is not the compiler that produced these artifacts; that is
# safe because cargo fingerprints the real compiler and rebuilds on a
# mismatch, it just means a base-image toolchain bump costs one cold build
# instead of invalidating the key.
- name: Cache cargo (docker builds)
uses: Swatinem/rust-cache@v2
if: ${{ matrix.settings.docker }}
with:
key: docker-${{ matrix.settings.target }}
cache-directories: .cargo-cache
save-if: ${{ github.ref == 'refs/heads/main' }}
path: |
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
.cargo-cache
target/
key: nodejs-${{ matrix.settings.target }}-cargo-${{ matrix.settings.host }}
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Install Zig
@@ -206,13 +225,9 @@ jobs:
if: ${{ matrix.settings.docker }}
with:
image: ${{ matrix.settings.docker }}
# All three mounts must live under `.cargo-cache`, which is what the
# cache step above saves. Previously the registry mounts pointed at
# `.cargo/...`, a path nothing cached, so the container re-downloaded
# the whole crate registry on every run.
options: "--user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db \
-v ${{ github.workspace }}/.cargo-cache/registry/cache:/usr/local/cargo/registry/cache \
-v ${{ github.workspace }}/.cargo-cache/registry/index:/usr/local/cargo/registry/index \
-v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache \
-v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index \
-v ${{ github.workspace }}:/build -w /build/nodejs"
run: |
set -e
@@ -224,16 +239,6 @@ jobs:
--js ../lancedb/native.js \
--strip \
--output-dir dist/
# The container runs as root (`--user 0:0`), so everything it wrote to the
# mounted cache dirs is root-owned. rust-cache's post step runs as the
# runner user and has to both read these and delete from them while
# pruning, so hand them back before it runs.
- name: Take ownership of docker build output
if: ${{ matrix.settings.docker }}
run: |
sudo chown -R "$(id -u):$(id -g)" \
"${{ github.workspace }}/.cargo-cache" \
"${{ github.workspace }}/target"
- name: Build
run: |
${{ matrix.settings.pre_build }}
@@ -247,15 +252,6 @@ jobs:
--output-dir dist/
if: ${{ !matrix.settings.docker }}
shell: bash
# The standard Windows runners have ~14 GB free, and a release `target/`
# for this workspace is a large fraction of that. Report the remaining
# headroom so a build that only just fits is visible before a dependency
# bump turns it into a failed release. `always()` so the numbers are
# still there when the build is what ran out of space.
- name: Report disk headroom
if: always()
run: df -h
shell: bash
- name: Upload artifact
uses: actions/upload-artifact@v7
with:
@@ -406,9 +402,7 @@ jobs:
name: Report Workflow Failure
runs-on: ubuntu-latest
needs: [build-lancedb, test-lancedb, publish]
# Nightly runs are the only thing watching the cross-compiled targets now,
# so they have to report failures too or the signal is silently lost.
if: always() && failure() && (startsWith(github.ref, 'refs/tags/v') || github.event_name == 'schedule')
if: always() && failure() && startsWith(github.ref, 'refs/tags/v')
permissions:
contents: read
issues: write
+72 -19
View File
@@ -3,7 +3,7 @@ name: PyPI Publish
on:
push:
tags:
- 'v*'
- 'python-v*'
pull_request:
# This should trigger a dry run (we skip the final publish step)
paths:
@@ -20,12 +20,6 @@ env:
permissions:
contents: read
# Without this, a force-push to a PR leaves the previous run going -- including
# a ~74 minute Windows job and a billed arm64 wheel build.
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
linux:
name: Python ${{ matrix.config.package_name }} ${{ matrix.config.platform }} manylinux${{ matrix.config.manylinux }}
@@ -78,7 +72,7 @@ jobs:
package-name: ${{ matrix.config.package_name }}
rustflags: ${{ matrix.config.rustflags }}
- uses: actions/upload-artifact@v7
if: startsWith(github.ref, 'refs/tags/v')
if: startsWith(github.ref, 'refs/tags/python-v')
with:
name: wheels-linux-${{ matrix.config.package_name }}-${{ matrix.config.platform }}-${{ matrix.config.manylinux }}
path: target/wheels/*.whl
@@ -107,7 +101,7 @@ jobs:
python-minor-version: 10
args: "--release --strip --target ${{ matrix.config.target }} --features fp16kernels"
- uses: actions/upload-artifact@v7
if: startsWith(github.ref, 'refs/tags/v')
if: startsWith(github.ref, 'refs/tags/python-v')
with:
name: wheels-mac-${{ matrix.config.target }}
path: target/wheels/lancedb-*.whl
@@ -128,26 +122,19 @@ jobs:
uses: actions/setup-python@v6
with:
python-version: "3.13"
# NOTE: caching cargo here would be a no-op. This workflow only runs on
# tags and PRs, and GitHub only lets a run restore caches from its own ref
# or the default branch -- so with no run on main there is nothing that
# can populate an entry the release build would be allowed to read. Fixing
# this needs a main/nightly trigger (which would also catch wheel-build
# breakage before a release); the ~74 minutes here is otherwise dominated
# by the fat-LTO link, which no cache avoids.
- uses: ./.github/workflows/build_windows_wheel
with:
python-minor-version: 10
args: "--release --strip"
- uses: actions/upload-artifact@v7
if: startsWith(github.ref, 'refs/tags/v')
if: startsWith(github.ref, 'refs/tags/python-v')
with:
name: wheels-windows
path: target/wheels/lancedb-*.whl
if-no-files-found: error
publish:
name: Publish wheels
if: startsWith(github.ref, 'refs/tags/v')
if: startsWith(github.ref, 'refs/tags/python-v')
needs: [linux, mac, windows]
runs-on: ubuntu-latest
permissions:
@@ -196,6 +183,72 @@ jobs:
uses: pypa/gh-action-pypi-publish@release/v1
with:
packages-dir: target/wheels/
gh-release:
if: startsWith(github.ref, 'refs/tags/python-v')
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
lfs: true
- name: Extract version
id: extract_version
env:
GITHUB_REF: ${{ github.ref }}
run: |
set -e
echo "Extracting tag and version from $GITHUB_REF"
if [[ $GITHUB_REF =~ refs/tags/python-v(.*) ]]; then
VERSION=${BASH_REMATCH[1]}
TAG=python-v$VERSION
echo "tag=$TAG" >> $GITHUB_OUTPUT
echo "version=$VERSION" >> $GITHUB_OUTPUT
else
echo "Failed to extract version from $GITHUB_REF"
exit 1
fi
echo "Extracted version $VERSION from $GITHUB_REF"
if [[ $VERSION =~ beta ]]; then
echo "This is a beta release"
# Get last release (that is not this one)
FROM_TAG=$(git tag --sort='version:refname' \
| grep ^python-v \
| grep -vF "$TAG" \
| python ci/semver_sort.py python-v \
| tail -n 1)
else
echo "This is a stable release"
# Get last stable tag (ignore betas)
FROM_TAG=$(git tag --sort='version:refname' \
| grep ^python-v \
| grep -vF "$TAG" \
| grep -v beta \
| python ci/semver_sort.py python-v \
| tail -n 1)
fi
echo "Found from tag $FROM_TAG"
echo "from_tag=$FROM_TAG" >> $GITHUB_OUTPUT
- name: Create Python Release Notes
id: python_release_notes
uses: mikepenz/release-changelog-builder-action@v4
with:
configuration: .github/release_notes.json
toTag: ${{ steps.extract_version.outputs.tag }}
fromTag: ${{ steps.extract_version.outputs.from_tag }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Create Python GH release
uses: softprops/action-gh-release@v2
with:
prerelease: ${{ contains('beta', github.ref) }}
tag_name: ${{ steps.extract_version.outputs.tag }}
token: ${{ secrets.GITHUB_TOKEN }}
generate_release_notes: false
name: Python LanceDB v${{ steps.extract_version.outputs.version }}
body: ${{ steps.python_release_notes.outputs.changelog }}
report-failure:
name: Report Workflow Failure
runs-on: ubuntu-latest
@@ -203,7 +256,7 @@ jobs:
permissions:
contents: read
issues: write
if: always() && failure() && startsWith(github.ref, 'refs/tags/v')
if: always() && failure() && startsWith(github.ref, 'refs/tags/python-v')
steps:
- uses: actions/checkout@v6
- uses: ./.github/actions/create-failure-issue
-33
View File
@@ -108,15 +108,6 @@ jobs:
run: |
sudo apt update
sudo apt install -y protobuf-compiler
# `pip install -e .` builds the extension with maturin, which is most of
# this job's ~33 minutes. It had no Rust cache, so every dependency was
# recompiled from scratch on every run.
- uses: Swatinem/rust-cache@v2
with:
# Restore everywhere, but only save from main. Per-PR saves are
# unreadable outside their own branch anyway, since GitHub scopes
# caches to the creating ref.
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Install
run: |
pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests,dev,embeddings]
@@ -177,14 +168,6 @@ jobs:
uses: actions/setup-python@v6
with:
python-version: "3.13"
# maturin runs cargo natively on macOS (docker is Linux-only), so the host
# target dir is cacheable. This job had no Rust cache.
- uses: Swatinem/rust-cache@v2
with:
# Restore everywhere, but only save from main. Per-PR saves are
# unreadable outside their own branch anyway, since GitHub scopes
# caches to the creating ref.
save-if: ${{ github.ref == 'refs/heads/main' }}
- uses: ./.github/workflows/build_mac_wheel
with:
args: --profile ci
@@ -214,14 +197,6 @@ jobs:
uses: actions/setup-python@v6
with:
python-version: "3.13"
# maturin runs cargo natively on Windows (docker is Linux-only), so the
# host target dir is cacheable. This job had no Rust cache at all and so
# rebuilt every dependency from scratch on every run.
- uses: Swatinem/rust-cache@v2
with:
# Restore everywhere, but only save from main. The repo sits at
# GitHub's cache cap, so per-PR saves just evict main's entries.
save-if: ${{ github.ref == 'refs/heads/main' }}
- uses: ./.github/workflows/build_windows_wheel
with:
args: --profile ci
@@ -249,14 +224,6 @@ jobs:
uses: actions/setup-python@v6
with:
python-version: "3.10"
# As with Doctest, `pip install -e .` compiles the extension and this job
# had no Rust cache, which is most of its ~37 minutes.
- uses: Swatinem/rust-cache@v2
with:
# Restore everywhere, but only save from main. Per-PR saves are
# unreadable outside their own branch anyway, since GitHub scopes
# caches to the creating ref.
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Install lancedb
run: |
pip install "pydantic<2"
+7 -45
View File
@@ -48,11 +48,6 @@ jobs:
with:
components: rustfmt, clippy
- uses: Swatinem/rust-cache@v2
with:
# Restore everywhere, but only save from main. Per-PR saves are
# unreadable outside their own branch anyway, since GitHub scopes
# caches to the creating ref.
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Install dependencies
run: |
sudo apt update
@@ -94,11 +89,6 @@ jobs:
run: rm -f Cargo.lock
- uses: rui314/setup-mold@v1
- uses: Swatinem/rust-cache@v2
with:
# Restore everywhere, but only save from main. Per-PR saves are
# unreadable outside their own branch anyway, since GitHub scopes
# caches to the creating ref.
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Install dependencies
run: |
sudo apt update
@@ -128,11 +118,6 @@ jobs:
fetch-depth: 0
lfs: true
- uses: Swatinem/rust-cache@v2
with:
# Restore everywhere, but only save from main. Per-PR saves are
# unreadable outside their own branch anyway, since GitHub scopes
# caches to the creating ref.
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Install dependencies
run: |
sudo apt update
@@ -190,11 +175,6 @@ jobs:
- name: CPU features
run: sysctl -a | grep cpu
- uses: Swatinem/rust-cache@v2
with:
# Restore everywhere, but only save from main. Per-PR saves are
# unreadable outside their own branch anyway, since GitHub scopes
# caches to the creating ref.
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Install dependencies
run: brew install protobuf
- name: Run tests
@@ -207,19 +187,12 @@ jobs:
cargo test --profile ci --features $ALL_FEATURES --locked
windows:
runs-on: windows-2022
strategy:
fail-fast: false
matrix:
include:
- target: x86_64-pc-windows-msvc
runner: windows-2022
# windows-11-arm is a standard runner, so it is free on public repos.
# Running natively lets the aarch64 tests actually execute -- this
# job used to cross-compile them and then skip the test step, paying
# full codegen and link cost for a compile check.
- target: aarch64-pc-windows-msvc
runner: windows-11-arm
runs-on: ${{ matrix.runner }}
target:
- x86_64-pc-windows-msvc
- aarch64-pc-windows-msvc
defaults:
run:
working-directory: rust/lancedb
@@ -228,11 +201,6 @@ jobs:
- name: Set target
run: rustup target add ${{ matrix.target }}
- uses: Swatinem/rust-cache@v2
with:
# Restore everywhere, but only save from main. Per-PR saves are
# unreadable outside their own branch anyway, since GitHub scopes
# caches to the creating ref.
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Install Protoc v21.12
run: choco install --no-progress protoc
- name: Build
@@ -240,12 +208,11 @@ jobs:
$env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
cargo build --profile ci --features aws,remote --tests --locked --target ${{ matrix.target }}
- name: Run tests
# Can only run tests when target matches host
if: ${{ matrix.target == 'x86_64-pc-windows-msvc' }}
run: |
$env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
# `--target` has to match the build step above. Without it cargo uses
# target/ci/ rather than target/<triple>/ci/ and rebuilds the entire
# dependency graph a second time.
cargo test --profile ci --features aws,remote --locked --target ${{ matrix.target }}
cargo test --profile ci --features aws,remote --locked
msrv:
# Check the minimum supported Rust version
@@ -271,11 +238,6 @@ jobs:
with:
toolchain: ${{ matrix.msrv }}
- uses: Swatinem/rust-cache@v2
with:
# Restore everywhere, but only save from main. Per-PR saves are
# unreadable outside their own branch anyway, since GitHub scopes
# caches to the creating ref.
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Downgrade dependencies
# These packages have newer requirements for MSRV
run: |
-29
View File
@@ -92,8 +92,6 @@ Python bindings changes:
* Should use `LOOP.run()` to call the corresponding `AsyncTable` method.
6. Add concrete sync method to `RemoteTable` class in `python/python/lancedb/remote/table.py`.
7. Add unit test in `python/tests/test_table.py`.
8. If you added a new public class or module-level function (not just a method on an
existing class), expose it in the API reference. See "Python API reference" below.
TypeScript bindings changes:
@@ -105,33 +103,6 @@ TypeScript bindings changes:
5. Add test in `nodejs/__test__/table.test.ts`.
6. Run `npm run docs` to generate TypeScript documentation.
## Python API reference
`docs/src/python/python.md` is the entire Python API reference. It is maintained by
hand, and anything not listed there is not rendered at all, so new public classes and
module-level functions have to be added explicitly. How depends on the module:
* `lancedb.index`, `lancedb.embeddings`, `lancedb.remote`, and `lancedb.rerankers` are
rendered by a single directive each, driven by the module's `__all__`. Add the new
name to `__all__` and it appears; forget, and it is silently omitted.
* Everything else (`lancedb`, `lancedb.table`, `lancedb.query`, `lancedb.db`, ...) is
listed symbol by symbol. Add a `::: lancedb.<module>.<Name>` line to the matching
section, and remember that the page separates synchronous and asynchronous APIs.
Deliberately undocumented: concrete implementations reached through an abstract base
(`LanceTable`, `LanceDBConnection`, `RemoteDBConnection`), query base classes already
covered by `inherited_members`, and internal helpers.
Cross-references in docstrings use mkdocstrings syntax, `[text][lancedb.table.Table]`.
Plain relative links such as `[Table](Table)` do not resolve. To check your work:
```shell
pip install -r docs/requirements.txt
cd docs && PYTHONPATH=. mkdocs build
```
The docs site only builds on pushes to `main`, so this is not covered by PR CI.
## Review Guidelines
Please consider the following when reviewing code contributions.
Generated
+151 -160
View File
@@ -217,9 +217,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]]
name = "arrow"
version = "58.4.0"
version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6cfdd0833e32a9874d2b55089333ad310c0be208aafa277385ce2461dec90be3"
checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -239,9 +239,9 @@ dependencies = [
[[package]]
name = "arrow-arith"
version = "58.4.0"
version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a"
checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -253,9 +253,9 @@ dependencies = [
[[package]]
name = "arrow-array"
version = "58.4.0"
version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69"
checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e"
dependencies = [
"ahash",
"arrow-buffer",
@@ -272,9 +272,9 @@ dependencies = [
[[package]]
name = "arrow-buffer"
version = "58.4.0"
version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5"
checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0"
dependencies = [
"bytes",
"half",
@@ -284,9 +284,9 @@ dependencies = [
[[package]]
name = "arrow-cast"
version = "58.4.0"
version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34"
checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -306,9 +306,9 @@ dependencies = [
[[package]]
name = "arrow-csv"
version = "58.4.0"
version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af0dd6d90d1955e9f9a014c1e563ee8aeffc21909085d25623e1da44d96eca26"
checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de"
dependencies = [
"arrow-array",
"arrow-cast",
@@ -321,9 +321,9 @@ dependencies = [
[[package]]
name = "arrow-data"
version = "58.4.0"
version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b"
checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0"
dependencies = [
"arrow-buffer",
"arrow-schema",
@@ -334,9 +334,9 @@ dependencies = [
[[package]]
name = "arrow-ipc"
version = "58.4.0"
version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29a908a11fcfb3fb2f6730f4ac15e367bc644e419155e96238f68cf3adde572b"
checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -350,9 +350,9 @@ dependencies = [
[[package]]
name = "arrow-json"
version = "58.4.0"
version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8a96aed3931c076adee39ec2a40d8219fc7f09e79bcdaca1df16272993e1e14"
checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -375,9 +375,9 @@ dependencies = [
[[package]]
name = "arrow-ord"
version = "58.4.0"
version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0"
checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -388,9 +388,9 @@ dependencies = [
[[package]]
name = "arrow-pyarrow"
version = "58.4.0"
version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ffb9be5a873590f825aef50df20e0f8dff5fd42a77058e28bbe7bd44bb53dec"
checksum = "d29abdf672a81c1aeb57fd2661457f9918964d49aed0e9f18932535f2a9e49ce"
dependencies = [
"arrow-array",
"arrow-data",
@@ -400,9 +400,9 @@ dependencies = [
[[package]]
name = "arrow-row"
version = "58.4.0"
version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "514ba0ef0d4c5896202dae736251ce415abb43a950bed570fb7981b8716c0e4c"
checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -413,9 +413,9 @@ dependencies = [
[[package]]
name = "arrow-schema"
version = "58.4.0"
version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e"
checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed"
dependencies = [
"bitflags 2.11.1",
"serde_core",
@@ -424,9 +424,9 @@ dependencies = [
[[package]]
name = "arrow-select"
version = "58.4.0"
version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad"
checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222"
dependencies = [
"ahash",
"arrow-array",
@@ -438,9 +438,9 @@ dependencies = [
[[package]]
name = "arrow-string"
version = "58.4.0"
version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6789b388467525e3271326b6b4915666ecfdf5142aef09779445c954b67543c"
checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -601,7 +601,7 @@ dependencies = [
"bytes",
"fastrand",
"hex",
"http 1.5.0",
"http 1.4.2",
"sha1 0.10.6",
"time",
"tokio",
@@ -664,7 +664,7 @@ dependencies = [
"bytes-utils",
"fastrand",
"http 0.2.12",
"http 1.5.0",
"http 1.4.2",
"http-body 0.4.6",
"http-body 1.1.0",
"percent-encoding",
@@ -694,7 +694,7 @@ dependencies = [
"bytes",
"fastrand",
"http 0.2.12",
"http 1.5.0",
"http 1.4.2",
"http-body-util",
"regex-lite",
"tracing",
@@ -719,7 +719,7 @@ dependencies = [
"bytes",
"fastrand",
"http 0.2.12",
"http 1.5.0",
"http 1.4.2",
"regex-lite",
"tracing",
]
@@ -743,7 +743,7 @@ dependencies = [
"bytes",
"fastrand",
"http 0.2.12",
"http 1.5.0",
"http 1.4.2",
"regex-lite",
"tracing",
]
@@ -773,7 +773,7 @@ dependencies = [
"hex",
"hmac 0.13.0",
"http 0.2.12",
"http 1.5.0",
"http 1.4.2",
"http-body 1.1.0",
"lru",
"percent-encoding",
@@ -802,7 +802,7 @@ dependencies = [
"bytes",
"fastrand",
"http 0.2.12",
"http 1.5.0",
"http 1.4.2",
"regex-lite",
"tracing",
]
@@ -826,7 +826,7 @@ dependencies = [
"bytes",
"fastrand",
"http 0.2.12",
"http 1.5.0",
"http 1.4.2",
"regex-lite",
"tracing",
]
@@ -851,7 +851,7 @@ dependencies = [
"aws-types",
"fastrand",
"http 0.2.12",
"http 1.5.0",
"http 1.4.2",
"regex-lite",
"tracing",
]
@@ -873,7 +873,7 @@ dependencies = [
"hex",
"hmac 0.13.0",
"http 0.2.12",
"http 1.5.0",
"http 1.4.2",
"p256",
"percent-encoding",
"ring",
@@ -906,7 +906,7 @@ dependencies = [
"bytes",
"crc-fast",
"hex",
"http 1.5.0",
"http 1.4.2",
"http-body 1.1.0",
"http-body-util",
"md-5 0.11.0",
@@ -940,7 +940,7 @@ dependencies = [
"bytes-utils",
"futures-core",
"futures-util",
"http 1.5.0",
"http 1.4.2",
"http-body 1.1.0",
"http-body-util",
"percent-encoding",
@@ -961,7 +961,7 @@ dependencies = [
"h2 0.3.27",
"h2 0.4.14",
"http 0.2.12",
"http 1.5.0",
"http 1.4.2",
"http-body 0.4.6",
"hyper 0.14.32",
"hyper 1.9.0",
@@ -1023,7 +1023,7 @@ dependencies = [
"bytes",
"fastrand",
"http 0.2.12",
"http 1.5.0",
"http 1.4.2",
"http-body 0.4.6",
"http-body 1.1.0",
"http-body-util",
@@ -1044,7 +1044,7 @@ dependencies = [
"aws-smithy-types",
"bytes",
"http 0.2.12",
"http 1.5.0",
"http 1.4.2",
"pin-project-lite",
"tokio",
"tracing",
@@ -1070,7 +1070,7 @@ checksum = "7442cb268338f0eb8278140a107c046756aa01093d8ef5e99628d34ae09c94f5"
dependencies = [
"aws-smithy-runtime-api",
"aws-smithy-types",
"http 1.5.0",
"http 1.4.2",
]
[[package]]
@@ -1084,7 +1084,7 @@ dependencies = [
"bytes-utils",
"futures-core",
"http 0.2.12",
"http 1.5.0",
"http 1.4.2",
"http-body 0.4.6",
"http-body 1.1.0",
"http-body-util",
@@ -1132,7 +1132,7 @@ dependencies = [
"axum-core",
"bytes",
"futures-util",
"http 1.5.0",
"http 1.4.2",
"http-body 1.1.0",
"http-body-util",
"hyper 1.9.0",
@@ -1165,7 +1165,7 @@ dependencies = [
"async-trait",
"bytes",
"futures-util",
"http 1.5.0",
"http 1.4.2",
"http-body 1.1.0",
"http-body-util",
"mime",
@@ -3421,8 +3421,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "10.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
version = "10.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
dependencies = [
"arrow-array",
"rand 0.9.5",
@@ -3842,7 +3842,7 @@ dependencies = [
"fnv",
"futures-core",
"futures-sink",
"http 1.5.0",
"http 1.4.2",
"indexmap 2.14.0",
"slab",
"tokio",
@@ -3953,7 +3953,7 @@ checksum = "629d8f3bbeda9d148036d6b0de0a3ab947abd08ce90626327fc3547a49d59d97"
dependencies = [
"dirs",
"futures",
"http 1.5.0",
"http 1.4.2",
"indicatif",
"libc",
"log",
@@ -3976,7 +3976,7 @@ checksum = "430b33fa84f92796d4d263070b6c0d3ca219df7b9a0e1853ee431029b1612bcd"
dependencies = [
"async-trait",
"bytes",
"http 1.5.0",
"http 1.4.2",
"more-asserts",
"serde",
"thiserror 2.0.18",
@@ -4041,9 +4041,9 @@ dependencies = [
[[package]]
name = "http"
version = "1.5.0"
version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0"
checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425"
dependencies = [
"bytes",
"itoa",
@@ -4067,7 +4067,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c"
dependencies = [
"bytes",
"http 1.5.0",
"http 1.4.2",
]
[[package]]
@@ -4078,7 +4078,7 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a"
dependencies = [
"bytes",
"futures-core",
"http 1.5.0",
"http 1.4.2",
"http-body 1.1.0",
"pin-project-lite",
]
@@ -4145,7 +4145,7 @@ dependencies = [
"futures-channel",
"futures-core",
"h2 0.4.14",
"http 1.5.0",
"http 1.4.2",
"http-body 1.1.0",
"httparse",
"httpdate",
@@ -4177,7 +4177,7 @@ version = "0.27.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f"
dependencies = [
"http 1.5.0",
"http 1.4.2",
"hyper 1.9.0",
"hyper-util",
"rustls 0.23.40",
@@ -4211,7 +4211,7 @@ dependencies = [
"bytes",
"futures-channel",
"futures-util",
"http 1.5.0",
"http 1.4.2",
"http-body 1.1.0",
"hyper 1.9.0",
"ipnet",
@@ -4777,8 +4777,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
[[package]]
name = "lance"
version = "10.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
version = "10.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
dependencies = [
"arc-swap",
"arrow",
@@ -4852,8 +4852,8 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "10.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
version = "10.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4875,7 +4875,7 @@ dependencies = [
[[package]]
name = "lance-arrow-scalar"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4889,7 +4889,7 @@ dependencies = [
[[package]]
name = "lance-arrow-stats"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -4898,8 +4898,8 @@ dependencies = [
[[package]]
name = "lance-bitpacking"
version = "10.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
version = "10.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
dependencies = [
"arrayref",
"crunchy",
@@ -4909,15 +4909,14 @@ dependencies = [
[[package]]
name = "lance-core"
version = "10.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
version = "10.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"async-trait",
"blake3",
"byteorder",
"bytes",
"datafusion-common",
@@ -4934,7 +4933,6 @@ dependencies = [
"object_store",
"pin-project",
"prost",
"quick_cache",
"rand 0.9.5",
"roaring",
"serde_json",
@@ -4950,8 +4948,8 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "10.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
version = "10.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
dependencies = [
"arrow",
"arrow-array",
@@ -4981,8 +4979,8 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "10.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
version = "10.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
dependencies = [
"arrow",
"arrow-array",
@@ -4999,8 +4997,8 @@ dependencies = [
[[package]]
name = "lance-derive"
version = "10.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
version = "10.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
dependencies = [
"proc-macro2",
"quote",
@@ -5009,8 +5007,8 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "10.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
version = "10.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5045,13 +5043,12 @@ dependencies = [
[[package]]
name = "lance-file"
version = "10.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
version = "10.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
dependencies = [
"arrow-arith",
"arrow-array",
"arrow-buffer",
"arrow-cast",
"arrow-data",
"arrow-schema",
"arrow-select",
@@ -5077,8 +5074,8 @@ dependencies = [
[[package]]
name = "lance-index"
version = "10.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
version = "10.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
dependencies = [
"arc-swap",
"arrow",
@@ -5145,8 +5142,8 @@ dependencies = [
[[package]]
name = "lance-index-core"
version = "10.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
version = "10.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5168,12 +5165,18 @@ dependencies = [
[[package]]
name = "lance-io"
version = "10.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
version = "10.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
dependencies = [
"arrow",
"arrow-arith",
"arrow-array",
"arrow-buffer",
"arrow-cast",
"arrow-data",
"arrow-schema",
"arrow-select",
"async-recursion",
"async-trait",
"aws-config",
"aws-credential-types",
@@ -5182,7 +5185,7 @@ dependencies = [
"chrono",
"futures",
"goosefs-sdk",
"http 1.5.0",
"http 1.4.2",
"io-uring",
"lance-arrow",
"lance-core",
@@ -5206,8 +5209,8 @@ dependencies = [
[[package]]
name = "lance-linalg"
version = "10.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
version = "10.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5223,8 +5226,8 @@ dependencies = [
[[package]]
name = "lance-namespace"
version = "10.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
version = "10.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
dependencies = [
"arrow",
"async-trait",
@@ -5236,8 +5239,8 @@ dependencies = [
[[package]]
name = "lance-namespace-impls"
version = "10.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
version = "10.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
dependencies = [
"arrow",
"arrow-ipc",
@@ -5291,8 +5294,8 @@ dependencies = [
[[package]]
name = "lance-select"
version = "10.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
version = "10.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5307,8 +5310,8 @@ dependencies = [
[[package]]
name = "lance-table"
version = "10.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
version = "10.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
dependencies = [
"arrow",
"arrow-array",
@@ -5347,8 +5350,8 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "10.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
version = "10.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5361,8 +5364,8 @@ dependencies = [
[[package]]
name = "lance-tokenizer"
version = "10.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
version = "10.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
dependencies = [
"icu_segmenter",
"jieba-rs",
@@ -5375,7 +5378,7 @@ dependencies = [
[[package]]
name = "lancedb"
version = "0.37.1-beta.0"
version = "0.32.0-beta.2"
dependencies = [
"ahash",
"anyhow",
@@ -5414,7 +5417,7 @@ dependencies = [
"goosefs-sdk",
"half",
"hf-hub",
"http 1.5.0",
"http 1.4.2",
"http-body 1.1.0",
"lance",
"lance-arrow",
@@ -5463,7 +5466,7 @@ dependencies = [
[[package]]
name = "lancedb-nodejs"
version = "0.37.1-beta.0"
version = "0.32.0-beta.2"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5488,7 +5491,7 @@ dependencies = [
[[package]]
name = "lancedb-python"
version = "0.37.1-beta.0"
version = "0.35.0-beta.2"
dependencies = [
"arrow",
"async-trait",
@@ -6080,15 +6083,15 @@ dependencies = [
[[package]]
name = "napi-build"
version = "2.4.0"
version = "2.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5282704fbe8d49b0cf8b08e3f33233416a528658f205c7e5ace63b582de0b11c"
checksum = "c9c366d2c8c60b86fa632df75f745509b52f9128f91a6bad4c796e44abb505e1"
[[package]]
name = "napi-derive"
version = "3.6.1"
version = "3.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d5c9c02556ea6dc99dffd36c1ce60141411657438501a125b675776d011ce92"
checksum = "a49c513341a61a16a10af6efcce46b30d0822ba2d4fb197d24d33dfc199c78d5"
dependencies = [
"convert_case",
"ctor 1.0.5",
@@ -6100,9 +6103,9 @@ dependencies = [
[[package]]
name = "napi-derive-backend"
version = "6.1.1"
version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d60b5d773ad46c698c8cc2cd9fde0b283d39cbb7f71c04bee633c7bdba4423bd"
checksum = "4747005fa3e2c9989ac45a723a514c5db2411238b72981a3cda4c701a9dfea17"
dependencies = [
"convert_case",
"proc-macro2",
@@ -6356,7 +6359,7 @@ dependencies = [
"futures-channel",
"futures-core",
"futures-util",
"http 1.5.0",
"http 1.4.2",
"http-body-util",
"httparse",
"humantime",
@@ -6477,7 +6480,7 @@ dependencies = [
"base64 0.22.1",
"bytes",
"futures",
"http 1.5.0",
"http 1.4.2",
"http-body 1.1.0",
"jiff",
"log",
@@ -6502,7 +6505,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d6f81ba6960e3fae1882f253b114b21d7e444e1534f209c7737a79f6243eb6f"
dependencies = [
"futures",
"http 1.5.0",
"http 1.4.2",
"mea",
"opendal-core",
]
@@ -6546,7 +6549,7 @@ checksum = "0030644366ef5d8cbe3a4a5822bf99a4aafddc1666e9d24b44d158d9062fc76a"
dependencies = [
"base64 0.22.1",
"bytes",
"http 1.5.0",
"http 1.4.2",
"log",
"opendal-core",
"opendal-service-azure-common",
@@ -6567,7 +6570,7 @@ checksum = "6dea4908d490143a9b0b7f7a790e139ff829b06a023f670455ed3d44f664b361"
dependencies = [
"base64 0.22.1",
"bytes",
"http 1.5.0",
"http 1.4.2",
"log",
"opendal-core",
"opendal-service-azure-common",
@@ -6585,7 +6588,7 @@ version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051"
dependencies = [
"http 1.5.0",
"http 1.4.2",
"opendal-core",
]
@@ -6596,7 +6599,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa8cafe9729213375c7331019b0cb756ad3e1aff7f45cd32c45eae91ebde8901"
dependencies = [
"bytes",
"http 1.5.0",
"http 1.4.2",
"log",
"opendal-core",
"quick-xml 0.39.4",
@@ -6614,7 +6617,7 @@ checksum = "48de101aac565ed06af4b47903c24eafd249075553ec1fb18256751c45148d47"
dependencies = [
"async-trait",
"bytes",
"http 1.5.0",
"http 1.4.2",
"log",
"opendal-core",
"percent-encoding",
@@ -6649,7 +6652,7 @@ checksum = "c4922661976a1d40794a2adfbdb888cc3c23097690f825a92f773af38908a848"
dependencies = [
"bytes",
"hf-xet",
"http 1.5.0",
"http 1.4.2",
"log",
"opendal-core",
"percent-encoding",
@@ -6665,7 +6668,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "328fa55e8888cbdfe00826bfea2a79042422b720e8369e9e021e46121dea5ace"
dependencies = [
"bytes",
"http 1.5.0",
"http 1.4.2",
"log",
"opendal-core",
"quick-xml 0.39.4",
@@ -6684,7 +6687,7 @@ dependencies = [
"base64 0.22.1",
"bytes",
"crc32c",
"http 1.5.0",
"http 1.4.2",
"log",
"md-5 0.11.0",
"opendal-core",
@@ -7583,7 +7586,7 @@ version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7"
dependencies = [
"heck 0.4.1",
"heck 0.5.0",
"itertools 0.14.0",
"log",
"multimap",
@@ -7799,18 +7802,6 @@ dependencies = [
"memchr",
]
[[package]]
name = "quick_cache"
version = "0.6.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9c6658afe513a3b484e3abfdaa0d03ef3c0bbf017542c178dd55f94eb3051f9"
dependencies = [
"ahash",
"equivalent",
"hashbrown 0.16.1",
"parking_lot",
]
[[package]]
name = "quinn"
version = "0.11.9"
@@ -8226,7 +8217,7 @@ checksum = "57ac2757f3140aa2e213b554148ae0b52733e624fc6723f0cc6bb3d440176c95"
dependencies = [
"anyhow",
"form_urlencoded",
"http 1.5.0",
"http 1.4.2",
"log",
"percent-encoding",
"reqsign-core",
@@ -8244,7 +8235,7 @@ dependencies = [
"anyhow",
"bytes",
"form_urlencoded",
"http 1.5.0",
"http 1.4.2",
"log",
"percent-encoding",
"quick-xml 0.39.4",
@@ -8266,7 +8257,7 @@ dependencies = [
"base64 0.22.1",
"bytes",
"form_urlencoded",
"http 1.5.0",
"http 1.4.2",
"jsonwebtoken",
"log",
"pem",
@@ -8291,7 +8282,7 @@ dependencies = [
"futures",
"hex",
"hmac 0.12.1",
"http 1.5.0",
"http 1.4.2",
"jiff",
"log",
"percent-encoding",
@@ -8318,7 +8309,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35cc609b49c69e76ecaceb775a03f792d1ed3e7755ab3548d4534fd801e3242e"
dependencies = [
"form_urlencoded",
"http 1.5.0",
"http 1.4.2",
"jsonwebtoken",
"log",
"percent-encoding",
@@ -8338,7 +8329,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e128f19525861dbded59e1e7c17653a8ed63d573ca04aed708d552dbef5bb32a"
dependencies = [
"anyhow",
"http 1.5.0",
"http 1.4.2",
"log",
"percent-encoding",
"reqsign-core",
@@ -8358,7 +8349,7 @@ dependencies = [
"futures-core",
"futures-util",
"h2 0.4.14",
"http 1.5.0",
"http 1.4.2",
"http-body 1.1.0",
"http-body-util",
"hyper 1.9.0",
@@ -8402,7 +8393,7 @@ dependencies = [
"bytes",
"futures-core",
"futures-util",
"http 1.5.0",
"http 1.4.2",
"http-body 1.1.0",
"http-body-util",
"hyper 1.9.0",
@@ -8456,7 +8447,7 @@ checksum = "199dda04a536b532d0cc04d7979e39b1c763ea749bf91507017069c00b96056f"
dependencies = [
"anyhow",
"async-trait",
"http 1.5.0",
"http 1.4.2",
"reqwest 0.13.3",
"thiserror 2.0.18",
"tower-service",
@@ -8498,9 +8489,9 @@ dependencies = [
[[package]]
name = "rkyv"
version = "0.8.17"
version = "0.8.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "815cc8a37159a463064825246cadb07961e25cd9885908606f6d08a98d8f8874"
checksum = "73389e0c99e664f919275ab5b5b0471391fe9a8de61e1dff9b1eaf56a90f16e3"
dependencies = [
"bytecheck",
"bytes",
@@ -8517,9 +8508,9 @@ dependencies = [
[[package]]
name = "rkyv_derive"
version = "0.8.17"
version = "0.8.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0ed1a78a1b19d184b0daa629dd9a024573173ec7d485b287cb369fb3607cc1c"
checksum = "5d2ed0b54125315fb36bd021e82d314d1c126548f871634b483f46b31d13cac6"
dependencies = [
"proc-macro2",
"quote",
@@ -9277,7 +9268,7 @@ version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451"
dependencies = [
"heck 0.4.1",
"heck 0.5.0",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -9289,7 +9280,7 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40"
dependencies = [
"heck 0.4.1",
"heck 0.5.0",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -9730,7 +9721,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
dependencies = [
"fastrand",
"getrandom 0.3.4",
"getrandom 0.4.2",
"once_cell",
"rustix",
"windows-sys 0.61.2",
@@ -10058,7 +10049,7 @@ dependencies = [
"base64 0.22.1",
"bytes",
"h2 0.4.14",
"http 1.5.0",
"http 1.4.2",
"http-body 1.1.0",
"http-body-util",
"hyper 1.9.0",
@@ -10114,7 +10105,7 @@ checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5"
dependencies = [
"bitflags 2.11.1",
"bytes",
"http 1.5.0",
"http 1.4.2",
"http-body 1.1.0",
"http-body-util",
"pin-project-lite",
@@ -10134,7 +10125,7 @@ dependencies = [
"bytes",
"futures-core",
"futures-util",
"http 1.5.0",
"http 1.4.2",
"http-body 1.1.0",
"http-body-util",
"pin-project-lite",
@@ -11155,7 +11146,7 @@ dependencies = [
"clap",
"crc32fast",
"futures",
"http 1.5.0",
"http 1.4.2",
"hyper 1.9.0",
"lazy_static",
"more-asserts",
@@ -11229,7 +11220,7 @@ dependencies = [
"chrono",
"clap",
"gearhash",
"http 1.5.0",
"http 1.4.2",
"itertools 0.14.0",
"lazy_static",
"more-asserts",
+14 -14
View File
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
rust-version = "1.91.0"
[workspace.dependencies]
lance = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance = { path = "../lance/rust/lance", default-features = false }
lance-core = { path = "../lance/rust/lance-core" }
lance-datagen = { path = "../lance/rust/lance-datagen" }
lance-file = { path = "../lance/rust/lance-file" }
lance-io = { path = "../lance/rust/lance-io", default-features = false }
lance-index = { path = "../lance/rust/lance-index" }
lance-linalg = { path = "../lance/rust/lance-linalg" }
lance-namespace = { path = "../lance/rust/lance-namespace" }
lance-namespace-impls = { path = "../lance/rust/lance-namespace-impls", default-features = false }
lance-table = { path = "../lance/rust/lance-table" }
lance-testing = { path = "../lance/rust/lance-testing" }
lance-datafusion = { path = "../lance/rust/lance-datafusion" }
lance-encoding = { path = "../lance/rust/lance-encoding" }
lance-arrow = { path = "../lance/rust/lance-arrow" }
ahash = "0.8"
# Note that this one does not include pyarrow
arrow = { version = "58.0.0", optional = false }
+3 -3
View File
@@ -2,9 +2,9 @@ set -e
RELEASE_TYPE=${1:-"stable"}
BUMP_MINOR=${2:-false}
HEAD_SHA=$(git rev-parse HEAD)
TAG_PREFIX=${3:-"v"} # Such as "python-v"
HEAD_SHA=${4:-$(git rev-parse HEAD)}
readonly TAG_PREFIX="v"
readonly SELF_DIR=$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
PREV_TAG=$(git tag --sort='version:refname' | grep ^$TAG_PREFIX | python $SELF_DIR/semver_sort.py $TAG_PREFIX | tail -n 1)
@@ -12,7 +12,7 @@ echo "Found previous tag $PREV_TAG"
# Initially, we don't want to tag if we are doing stable, because we will bump
# again later. See comment at end for why.
if [[ "$RELEASE_TYPE" == 'stable' ]]; then
if [[ "$RELEASE_TYPE" == 'stable' ]]; then
BUMP_ARGS="--no-tag"
fi
-5
View File
@@ -51,11 +51,6 @@ plugins:
paths: [../python/python]
options:
docstring_style: numpy
docstring_options:
# Attributes documented in a `Parameters` section, and pydantic
# dataclasses whose `__init__` griffe cannot see statically, both
# trip this check. It reports nothing actionable here.
warn_unknown_params: false
heading_level: 3
show_signature_annotations: true
show_root_heading: true
+1 -11
View File
@@ -453,16 +453,6 @@ paths:
The metric type to use for the index. l2, Cosine, Dot are supported.
index_type:
type: string
custom_stop_words:
type: [array, "null"]
items:
type: string
description: |
The custom stop-word list for an FTS index. A non-null
array replaces the language's built-in stop-word list and is only
applied when remove_stop_words is enabled. Null uses the built-in
language list, while an empty array explicitly replaces it with no
stop words.
responses:
"200":
description: Index successfully created
@@ -520,4 +510,4 @@ paths:
"401":
$ref: "#/components/responses/unauthorized"
"404":
$ref: "#/components/responses/not_found"
$ref: "#/components/responses/not_found"
+1 -1
View File
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
<dependency>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-core</artifactId>
<version>0.37.1-beta.0</version>
<version>0.32.0-beta.2</version>
</dependency>
```
+1 -1
View File
@@ -1,7 +1,7 @@
# Contributing to LanceDB Typescript
This document outlines the process for contributing to LanceDB Typescript.
For general contribution guidelines, see [CONTRIBUTING.md](https://github.com/lancedb/lancedb/blob/main/CONTRIBUTING.md).
For general contribution guidelines, see [CONTRIBUTING.md](../CONTRIBUTING.md).
## Project layout
-97
View File
@@ -25,27 +25,6 @@ the underlying connection has been closed.
## Methods
### cancelJob()
```ts
abstract cancelJob(jobId): Promise<boolean>
```
Request cancellation of a server-side job by id.
Resolves to true if the server accepted the cancellation, false if no
such job exists. Cancelling an already-terminal job is a no-op success.
#### Parameters
* **jobId**: `string`
#### Returns
`Promise`&lt;`boolean`&gt;
***
### cloneTable()
```ts
@@ -386,26 +365,6 @@ Drop an existing table.
***
### getJob()
```ts
abstract getJob(jobId): Promise<null | JobDescription>
```
Describe a single server-side job by id.
Resolves to `null` when the server has no such job.
#### Parameters
* **jobId**: `string`
#### Returns
`Promise`&lt;`null` \| [`JobDescription`](../interfaces/JobDescription.md)&gt;
***
### isOpen()
```ts
@@ -420,62 +379,6 @@ Return true if the connection has not been closed
***
### job()
```ts
abstract job(jobId): Job
```
A [Job](Job.md) handle for a server-side job by id.
The handle is constructed without a server round trip; an unknown id
surfaces when the handle is used. Dropping the handle has no effect on
the job itself.
#### Parameters
* **jobId**: `string`
#### Returns
[`Job`](Job.md)
***
### jobHistory()
```ts
abstract jobHistory(jobId?): Promise<Table<any>>
```
The lifecycle event history of a server-side job, as an Arrow table.
Lists history across all jobs when `jobId` is omitted.
#### Parameters
* **jobId?**: `string`
#### Returns
`Promise`&lt;`Table`&lt;`any`&gt;&gt;
***
### listJobs()
```ts
abstract listJobs(): Promise<JobInfo[]>
```
List server-side jobs across the database's tables.
#### Returns
`Promise`&lt;[`JobInfo`](../interfaces/JobInfo.md)[]&gt;
***
### listNamespaces()
```ts
-83
View File
@@ -1,83 +0,0 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / Job
# Class: Job
A handle to an operation that may still be running.
## Constructors
### new Job()
```ts
new Job(): Job
```
#### Returns
[`Job`](Job.md)
## Accessors
### id
```ts
get id(): null | string
```
Identifies the operation on the server that is running it. Operations
that run in this process have no server id. The value is opaque.
#### Returns
`null` \| `string`
## Methods
### cancel()
```ts
cancel(): Promise<void>
```
Request cancellation. Cancelling a finished operation is a no-op.
#### Returns
`Promise`&lt;`void`&gt;
***
### status()
```ts
status(): Promise<string>
```
The operation's current lifecycle state: "running", "finished",
"failed", or "cancelled".
A point snapshot; unlike [Job.wait](Job.md#wait) it does not block or reject
on a terminal failure state. States a newer server reports that this
client version does not know pass through as-is.
#### Returns
`Promise`&lt;`string`&gt;
***
### wait()
```ts
wait(): Promise<void>
```
Wait until the operation reaches a terminal state.
#### Returns
`Promise`&lt;`void`&gt;
+9 -8
View File
@@ -76,23 +76,24 @@ the query optimizer chooses a suboptimal path.
***
### useLsm()
### useLsmWrite()
```ts
useLsm(enable): MergeInsertBuilder
useLsmWrite(useLsmWrite): MergeInsertBuilder
```
Control MemWAL routing for this merge.
Controls whether the merge uses the MemWAL LSM write path.
By default (unset), a `mergeInsert` on a table with an LSM write spec is
routed through Lance's MemWAL shard writer, and a table without one uses the
standard path.
routed through Lance's MemWAL shard writer, and a table without one uses
the standard path. Pass `false` to force the standard path even when a
spec is set. Pass `true` to require a spec — `mergeInsert` rejects if none
is installed.
#### Parameters
* **enable**: `boolean`
`true` forces MemWAL routing and errors if the table has no
LSM write spec. `false` forces the standard write path even when a spec is set.
* **useLsmWrite**: `boolean`
Whether to use the LSM write path.
#### Returns
-36
View File
@@ -497,42 +497,6 @@ ArrowTable.
***
### useLsm()
```ts
useLsm(enable): this
```
Control MemWAL read routing for this query.
By default (unset), when the table carries a MemWAL write spec (see
[Table#setLsmWriteSpec](Table.md#setlsmwritespec)), reads are routed through the LSM scanner so
they also return data written via the `mergeInsert` LSM path that has not yet
been compacted into the base table (the active/frozen in-memory memtables and
the flushed generations), deduplicated by primary key; a table without a spec
reads the base table.
#### Parameters
* **enable**: `boolean`
`true` forces the LSM scanner and errors if the table has no
MemWAL write spec. `false` bypasses the MemWAL and reads the base table only,
even when a spec is present.
Note: the LSM scanner does not support every query shape (e.g. reranking,
hybrid search, `orderBy`). On a MemWAL table those shapes error unless
`useLsm(false)` is set, because a base-only read would silently exclude
un-compacted MemWAL data.
#### Returns
`this`
#### Inherited from
`StandardQueryBase.useLsm`
***
### where()
```ts
-23
View File
@@ -295,29 +295,6 @@ await table.createIndex("my_float_col");
***
### createIndexAsync()
```ts
abstract createIndexAsync(column, options?): Promise<Job>
```
Create an index, returning a handle to the indexing job.
The job may already be complete when returned; callers must not assume
the index exists until [Job.wait](Job.md#wait) resolves.
#### Parameters
* **column**: `string`
* **options?**: `Partial`&lt;[`IndexOptions`](../interfaces/IndexOptions.md)&gt;
#### Returns
`Promise`&lt;[`Job`](Job.md)&gt;
***
### currentBranch()
```ts
-23
View File
@@ -273,29 +273,6 @@ ArrowTable.
***
### useLsm()
```ts
useLsm(enable): this
```
Control MemWAL read routing for this take query.
`false` bypasses the MemWAL and reads the base table only — the escape hatch,
since take-by-row-id/offset is not supported on the LSM scanner and, on a
MemWAL table, auto-routes to it and errors otherwise.
#### Parameters
* **enable**: `boolean`
`false` reads the base table only.
#### Returns
`this`
***
### withRowId()
```ts
-36
View File
@@ -746,42 +746,6 @@ ArrowTable.
***
### useLsm()
```ts
useLsm(enable): this
```
Control MemWAL read routing for this query.
By default (unset), when the table carries a MemWAL write spec (see
[Table#setLsmWriteSpec](Table.md#setlsmwritespec)), reads are routed through the LSM scanner so
they also return data written via the `mergeInsert` LSM path that has not yet
been compacted into the base table (the active/frozen in-memory memtables and
the flushed generations), deduplicated by primary key; a table without a spec
reads the base table.
#### Parameters
* **enable**: `boolean`
`true` forces the LSM scanner and errors if the table has no
MemWAL write spec. `false` bypasses the MemWAL and reads the base table only,
even when a spec is present.
Note: the LSM scanner does not support every query shape (e.g. reranking,
hybrid search, `orderBy`). On a MemWAL table those shapes error unless
`useLsm(false)` is set, because a base-only read would silently exclude
un-compacted MemWAL data.
#### Returns
`this`
#### Inherited from
`StandardQueryBase.useLsm`
***
### where()
```ts
-4
View File
@@ -25,7 +25,6 @@
- [Connection](classes/Connection.md)
- [HeaderProvider](classes/HeaderProvider.md)
- [Index](classes/Index.md)
- [Job](classes/Job.md)
- [MakeArrowTableOptions](classes/MakeArrowTableOptions.md)
- [MatchQuery](classes/MatchQuery.md)
- [MergeInsertBuilder](classes/MergeInsertBuilder.md)
@@ -89,9 +88,6 @@
- [IvfFlatOptions](interfaces/IvfFlatOptions.md)
- [IvfPqOptions](interfaces/IvfPqOptions.md)
- [IvfRqOptions](interfaces/IvfRqOptions.md)
- [JobDescription](interfaces/JobDescription.md)
- [JobFailureInfo](interfaces/JobFailureInfo.md)
- [JobInfo](interfaces/JobInfo.md)
- [ListNamespacesOptions](interfaces/ListNamespacesOptions.md)
- [ListNamespacesResponse](interfaces/ListNamespacesResponse.md)
- [LsmWriteSpec](interfaces/LsmWriteSpec.md)
-28
View File
@@ -43,34 +43,6 @@ The following tokenizers are available:
***
### blockSize?
```ts
optional blockSize: 128 | 256;
```
Number of documents per compressed posting block.
The default is 128. Supported values are 128 and 256. A value of 256 uses
the experimental FTS V3 format and may introduce breaking changes.
***
### customStopWords?
```ts
optional customStopWords: string[];
```
Custom stop words that replace the built-in list for `language`.
This option only affects tokenization when `removeStopWords` is true.
`undefined` keeps the built-in language list. An empty array explicitly
replaces it with no stop words.
***
### language?
```ts
-66
View File
@@ -1,66 +0,0 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / JobDescription
# Interface: JobDescription
A described job from `Connection.getJob`.
## Properties
### creationMs
```ts
creationMs: number;
```
When the job was created, in milliseconds since the epoch.
***
### failure?
```ts
optional failure: JobFailureInfo;
```
Why the job failed, when the job is failed and the server reports a
reason.
***
### jobId
```ts
jobId: string;
```
***
### jobType
```ts
jobType: string;
```
***
### specJson?
```ts
optional specJson: string;
```
The job-type-specific specification as a JSON string, when present.
***
### state
```ts
state: string;
```
Lifecycle state: "running", "finished", "failed", or "cancelled".
-33
View File
@@ -1,33 +0,0 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / JobFailureInfo
# Interface: JobFailureInfo
The server's account of why a job failed.
## Properties
### message?
```ts
optional message: string;
```
***
### phase?
```ts
optional phase: string;
```
***
### retryable?
```ts
optional retryable: boolean;
```
-58
View File
@@ -1,58 +0,0 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / JobInfo
# Interface: JobInfo
A row from `Connection.listJobs`: one server-side job.
## Properties
### createdAtMillis
```ts
createdAtMillis: number;
```
When the job was created, in milliseconds since the epoch.
***
### jobId
```ts
jobId: string;
```
The job id -- what `Connection.getJob` and `Connection.cancelJob`
accept.
***
### jobType
```ts
jobType: string;
```
***
### state
```ts
state: string;
```
Lifecycle state: "running", "finished", "failed", or "cancelled".
***
### table
```ts
table: string;
```
The table the job runs against, without URI or namespace.
-15
View File
@@ -30,21 +30,6 @@ The tokenizer to use. The default is "simple".
***
### customStopWords?
```ts
optional customStopWords: string[];
```
Custom stop words that replace the built-in list for `language`.
This option only affects tokenization when `removeStopWords` is true.
`undefined` keeps the built-in language list. An empty array explicitly
replaces it with no stop words.
***
### language?
```ts
+52 -141
View File
@@ -26,18 +26,6 @@ is also an [asynchronous API client](#connections-asynchronous).
::: lancedb.db.DBConnection
::: lancedb.Session
## Namespaces (Synchronous)
A namespace-backed connection resolves tables through a
[Lance namespace](https://lancedb.github.io/lance-namespace/) service instead of
listing a storage directory.
::: lancedb.connect_namespace
::: lancedb.namespace.LanceNamespaceDBConnection
## Tables (Synchronous)
::: lancedb.table.Table
@@ -46,12 +34,8 @@ listing a storage directory.
::: lancedb.table.FragmentSummaryStats
::: lancedb.table.TableStatistics
::: lancedb.table.Tags
::: lancedb.table.Branches
## Expressions
Type-safe expression builder for filters and projections. Use these instead
@@ -78,46 +62,29 @@ of raw SQL strings with [where][lancedb.query.LanceQueryBuilder.where] and
::: lancedb.query.LanceHybridQueryBuilder
::: lancedb.query.LanceEmptyQueryBuilder
::: lancedb.query.LanceTakeQueryBuilder
## Full text queries
Structured full text queries can be passed to
[Table.search][lancedb.table.Table.search] or
[AsyncTable.search][lancedb.table.AsyncTable.search] in place of a query string,
and combined with [BooleanQuery][lancedb.query.BooleanQuery].
::: lancedb.query.FullTextQuery
::: lancedb.query.MatchQuery
::: lancedb.query.PhraseQuery
::: lancedb.query.BoostQuery
::: lancedb.query.MultiMatchQuery
::: lancedb.query.BooleanQuery
::: lancedb.query.FullTextOperator
::: lancedb.query.Occur
## Embeddings
::: lancedb.embeddings
options:
show_root_heading: false
show_root_toc_entry: false
::: lancedb.embeddings.registry.EmbeddingFunctionRegistry
::: lancedb.embeddings.base.EmbeddingFunctionConfig
::: lancedb.embeddings.base.EmbeddingFunction
::: lancedb.embeddings.base.TextEmbeddingFunction
::: lancedb.embeddings.sentence_transformers.SentenceTransformerEmbeddings
::: lancedb.embeddings.openai.OpenAIEmbeddings
::: lancedb.embeddings.open_clip.OpenClipEmbeddings
## Remote configuration
::: lancedb.remote
options:
show_root_heading: false
show_root_toc_entry: false
::: lancedb.remote.ClientConfig
::: lancedb.remote.TimeoutConfig
::: lancedb.remote.RetryConfig
## Context
@@ -127,50 +94,11 @@ and combined with [BooleanQuery][lancedb.query.BooleanQuery].
## Full text search
Pass `custom_stop_words` to [lancedb.index.FTS][]:
Use [lancedb.table.Table.create_fts_index][] for the synchronous API or
[lancedb.table.AsyncTable.create_index][] with [lancedb.index.FTS][] for the
asynchronous API.
```python
from lancedb.index import FTS
table.create_index(
"text",
config=FTS(remove_stop_words=True, custom_stop_words=["acme", "internal"]),
)
```
The list replaces the built-in stop words and is used only when
`remove_stop_words=True`:
- `custom_stop_words=None` uses the built-in list for `language`.
- `custom_stop_words=[]` removes no words.
- Values are passed through without trimming, lowercasing, or other rewriting.
The same option is available on `lancedb.tokenize(...)` and the deprecated
[lancedb.table.Table.create_fts_index][] compatibility helper:
```python
import lancedb
tokens = list(lancedb.tokenize("acme makes searchable data",
custom_stop_words=["acme"]))
```
::: lancedb.tokenize
::: lancedb.FtsToken
## Blobs
Blob columns store large binary values out of line so they can be read lazily
instead of being materialized with the rest of the row.
::: lancedb.blob
::: lancedb.BlobType
::: lancedb._blob.BlobFile
options:
show_root_full_path: false
::: lancedb.index.FTS
## Utilities
@@ -178,14 +106,6 @@ instead of being materialized with the rest of the row.
::: lancedb.merge.LanceMergeInsertBuilder
::: lancedb.otel.instrument_lancedb_metrics
## Exceptions
::: lancedb.exceptions.MissingValueError
::: lancedb.exceptions.MissingColumnError
## Integrations
## Pydantic
@@ -194,30 +114,19 @@ instead of being materialized with the rest of the row.
::: lancedb.pydantic.vector
::: lancedb.pydantic.Vector
::: lancedb.pydantic.MultiVector
::: lancedb.pydantic.LanceModel
## PyTorch
::: lancedb.streaming.StreamingDataset
::: lancedb.permutation.permutation_builder
::: lancedb.permutation.PermutationBuilder
::: lancedb.permutation.Permutation
::: lancedb.permutation.Transforms
## Reranking
::: lancedb.rerankers
options:
show_root_heading: false
show_root_toc_entry: false
::: lancedb.rerankers.linear_combination.LinearCombinationReranker
::: lancedb.rerankers.cohere.CohereReranker
::: lancedb.rerankers.colbert.ColbertReranker
::: lancedb.rerankers.cross_encoder.CrossEncoderReranker
::: lancedb.rerankers.openai.OpenaiReranker
## Connections (Asynchronous)
@@ -228,12 +137,6 @@ can be used to create, list, or open tables.
::: lancedb.db.AsyncConnection
## Namespaces (Asynchronous)
::: lancedb.connect_namespace_async
::: lancedb.namespace.AsyncLanceNamespaceDBConnection
## Tables (Asynchronous)
Table hold your actual data as a collection of records / rows.
@@ -242,20 +145,32 @@ Table hold your actual data as a collection of records / rows.
::: lancedb.table.AsyncTags
::: lancedb.table.AsyncBranches
## Indices (Asynchronous)
Indices can be created on a table to speed up queries. This section
lists the indices that LanceDb supports.
::: lancedb.index
options:
show_root_heading: false
show_root_toc_entry: false
# `lang_mapping` is defined in the module rather than imported, so it is
# picked up despite not being in `__all__`. It is an internal lookup table.
filters: ["!^_", "!^lang_mapping$"]
::: lancedb.index.BTree
::: lancedb.index.Bitmap
::: lancedb.index.LabelList
::: lancedb.index.FTS
::: lancedb.index.IvfPq
::: lancedb.index.HnswPq
::: lancedb.index.HnswSq
::: lancedb.index.IvfFlat
::: lancedb.index.IvfSq
::: lancedb.index.IvfRq
::: lancedb.index.HnswFlat
::: lancedb.table.IndexStatistics
@@ -283,7 +198,3 @@ rows nearest to a query vector and can be created with the
::: lancedb.query.AsyncHybridQuery
options:
inherited_members: true
::: lancedb.query.AsyncTakeQuery
options:
inherited_members: true
+1 -1
View File
@@ -8,7 +8,7 @@
<parent>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.37.1-beta.0</version>
<version>0.32.0-beta.2</version>
<relativePath>../pom.xml</relativePath>
</parent>
+2 -2
View File
@@ -6,7 +6,7 @@
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.37.1-beta.0</version>
<version>0.32.0-beta.2</version>
<packaging>pom</packaging>
<name>${project.artifactId}</name>
<description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<arrow.version>15.0.0</arrow.version>
<lance-core.version>10.1.0-beta.1</lance-core.version>
<lance-core.version>10.0.0-beta.2</lance-core.version>
<spotless.skip>false</spotless.skip>
<spotless.version>2.30.0</spotless.version>
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
+1 -1
View File
@@ -1,7 +1,7 @@
# Contributing to LanceDB Typescript
This document outlines the process for contributing to LanceDB Typescript.
For general contribution guidelines, see [CONTRIBUTING.md](https://github.com/lancedb/lancedb/blob/main/CONTRIBUTING.md).
For general contribution guidelines, see [CONTRIBUTING.md](../CONTRIBUTING.md).
## Project layout
+1 -1
View File
@@ -1,7 +1,7 @@
[package]
name = "lancedb-nodejs"
edition.workspace = true
version = "0.37.1-beta.0"
version = "0.32.0-beta.2"
publish = false
license.workspace = true
description.workspace = true
-31
View File
@@ -991,37 +991,6 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
expectValidMapField(roundTripped.schema.fields[0]);
});
it("preserves string schema metadata", function () {
const metadata = new Map([["source", "fixture"]]);
const schema = new Schema(
[new Field("value", new Int32(), true)],
metadata,
);
expect(makeEmptyTable(schema).schema.metadata.get("source")).toBe(
"fixture",
);
});
it.each([
["non-string keys", new Map<unknown, unknown>([[42, "fixture"]])],
["non-string values", new Map<unknown, unknown>([["source", 42]])],
[
"non-string keys and values",
new Map<unknown, unknown>([[42, false]]),
],
])("rejects schema metadata with %s", function (_, metadataLike) {
const metadata = metadataLike as unknown as Map<string, string>;
const schema = new Schema(
[new Field("value", new Int32(), true)],
metadata,
);
expect(() => makeEmptyTable(schema)).toThrow(
"Expected metadata, if present, to be a Map<string, string> but it had non-string keys or values",
);
});
});
describe("when using two versions of arrow", function () {
-147
View File
@@ -15,7 +15,6 @@ import {
OAuthHeaderProvider,
StaticHeaderProvider,
} from "../lancedb/header";
import { Index } from "../lancedb/indices";
// Test-only header providers
class CustomProvider extends HeaderProvider {
@@ -226,59 +225,6 @@ describe("remote connection", () => {
);
});
it("sends FTS options to remote tables", async () => {
let createIndexBody: Record<string, unknown> | undefined;
await withMockDatabase(
(req, res) => {
const path = req.url ?? "";
if (path.endsWith("/describe/")) {
res.writeHead(200, { "Content-Type": "application/json" }).end(
JSON.stringify({
name: "t",
version: 1,
schema: {
fields: [
{ name: "text", type: { type: "string" }, nullable: false },
],
},
}),
);
return;
}
if (path.endsWith("/create_index/")) {
let raw = "";
req.on("data", (chunk) => {
raw += chunk;
});
req.on("end", () => {
createIndexBody = JSON.parse(raw);
res.writeHead(200).end();
});
return;
}
res.writeHead(404).end();
},
async (db) => {
const table = await db.openTable("t");
await table.createIndex("text", {
config: Index.fts({
blockSize: 256,
removeStopWords: true,
customStopWords: ["the"],
}),
});
},
);
expect(createIndexBody?.["column"]).toBe("text");
expect(createIndexBody?.["index_type"]).toBe("FTS");
expect(createIndexBody?.["block_size"]).toBe(256);
expect(createIndexBody?.["custom_stop_words"]).toEqual(["the"]);
});
it("diffs and merges remote branches", async () => {
const sampleDiff = {
fromBranch: "exp",
@@ -877,96 +823,3 @@ describe("remote connection", () => {
});
});
});
describe("remote connection jobs surface", () => {
it("lists, describes, cancels, and reads history", async () => {
const { tableFromArrays, tableToIPC } = await import("apache-arrow");
const eventsTable = tableFromArrays({ state: ["created", "succeeded"] });
const eventsBody = Buffer.from(tableToIPC(eventsTable, "stream"));
await withMockDatabase(
(req, res) => {
let body = "";
req.on("data", (chunk) => {
body += chunk;
});
req.on("end", () => {
const payload = body.length > 0 ? JSON.parse(body) : {};
if (req.url === "/v1/jobs/list") {
if (payload["page_token"] === undefined) {
res
.writeHead(200, { "Content-Type": "application/json" })
.end(
'{"jobs": [{"job_id": "job-1", "table": "t1", ' +
'"job_type": "create_index", "state": "in_progress", ' +
'"created_at_millis": 1000}], "page_token": "next"}',
);
} else {
res
.writeHead(200, { "Content-Type": "application/json" })
.end(
'{"jobs": [{"job_id": "job-2", "table": "t2", ' +
'"job_type": "create_index", "state": "succeeded", ' +
'"created_at_millis": 2000}]}',
);
}
} else if (req.url === "/v1/jobs/describe") {
if (payload["job_id"] !== "job-1") {
res.writeHead(404).end("no such job");
return;
}
res
.writeHead(200, { "Content-Type": "application/json" })
.end(
'{"job_id": "job-1", "job_type": "create_index", ' +
'"job_state": "FAILED", "creation_ms": 1000, ' +
'"spec": {"column": "vec"}, "failure": {"phase": "execute", ' +
'"message": "worker died", "retryable": true}}',
);
} else if (req.url === "/v1/jobs/cancel") {
if (payload["job_id"] !== "job-1") {
res.writeHead(404).end("no such job");
return;
}
res
.writeHead(200, { "Content-Type": "application/json" })
.end('{"job_id": "job-1"}');
} else if (req.url === "/v1/jobs/query_events") {
res
.writeHead(200, {
"Content-Type": "application/vnd.apache.arrow.stream",
})
.end(eventsBody);
} else {
res.writeHead(404).end();
}
});
},
async (db) => {
const jobs = await db.listJobs();
expect(jobs.map((job) => job.jobId)).toEqual(["job-1", "job-2"]);
expect(jobs[0].state).toEqual("running");
expect(jobs[1].state).toEqual("finished");
const description = await db.getJob("job-1");
expect(description?.state).toEqual("failed");
expect(JSON.parse(description?.specJson ?? "")).toEqual({
column: "vec",
});
expect(description?.failure?.message).toEqual("worker died");
expect(await db.getJob("missing")).toBeNull();
expect(await db.cancelJob("job-1")).toBe(true);
expect(await db.cancelJob("missing")).toBe(false);
const history = await db.jobHistory("job-1");
expect(history.numRows).toEqual(2);
const job = db.job("job-1");
expect(job.id).toEqual("job-1");
expect(await job.status()).toEqual("failed");
await expect(job.wait()).rejects.toThrow("worker died");
},
);
});
});
+3 -85
View File
@@ -527,14 +527,6 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
);
});
it("should expose useLsm on takeRowIds as the base-only escape hatch", async () => {
await table.add([{ id: 1 }, { id: 2 }, { id: 3 }]);
// useLsm(false) is reachable on TakeQuery (the escape hatch for MemWAL tables,
// where take-by-row-id auto-routes to the LSM scanner and is rejected).
const res = await table.takeRowIds([0, 2]).useLsm(false).toArray();
expect(res.map((r) => r.id)).toEqual([1, 3]);
});
it("should throw for negative number in takeRowIds", () => {
expect(() => table.takeRowIds([-1])).toThrow("Row id cannot be negative");
expect(() => table.takeRowIds([0, -5, 2])).toThrow(
@@ -851,11 +843,7 @@ describe("When creating an index", () => {
afterEach(() => tmpDir.removeCallback());
it("should create a vector index on vector columns", async () => {
const job = await tbl.createIndexAsync("vec");
expect(job.id).toBeNull();
await job.wait();
// Cancelling a job that already finished succeeds and does nothing.
await job.cancel();
await tbl.createIndex("vec");
// check index directory
const indexDir = path.join(tmpDir.name, "test.lance", "_indices");
@@ -2539,35 +2527,6 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
expect(results3.length).toBe(1);
});
test("full text search with custom posting block size", async () => {
const db = await connect(tmpDir.name);
const data = [
{ text: "hello world", vector: [0.1, 0.2, 0.3] },
{ text: "goodbye world", vector: [0.4, 0.5, 0.6] },
];
const table = await db.createTable("test", data);
await table.createIndex("text", {
config: Index.fts({ blockSize: 256 }),
});
const index = (await table.listIndices()).find(
(index) => index.indexType === "FTS",
);
expect(index?.indexVersion).toBe(3);
expect(
(index?.indexDetails as Record<string, unknown>)["block_size"],
).toBe(256);
const results = await table.search("hello").toArray();
expect(results[0].text).toBe(data[0].text);
});
test("rejects invalid full text posting block size", () => {
expect(() => Index.fts({ blockSize: 129 as 128 | 256 })).toThrow(
"128 or 256",
);
});
test("full text search without lowercase", async () => {
const db = await connect(tmpDir.name);
const data = [
@@ -2773,15 +2732,6 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
},
);
test("tokenize supports custom stop words", async () => {
const tokens = await tokenize("the lance data", {
stem: false,
removeStopWords: true,
customStopWords: ["lance"],
});
expect(tokens.map((token) => token.text)).toEqual(["the", "data"]);
});
describe("when calling explainPlan", () => {
let tmpDir: tmp.DirResult;
let table: Table;
@@ -3220,14 +3170,14 @@ describe("LSM merge insert", () => {
await table.closeLsmWriters();
});
it("falls back to the standard path with useLsm(false)", async () => {
it("falls back to the standard path with useLsmWrite(false)", async () => {
const conn = await connect(tmpDir.name);
const table = await bucketTable(conn);
const res = await table
.mergeInsert("id")
.whenNotMatchedInsertAll()
.useLsm(false)
.useLsmWrite(false)
.execute([
{ id: "b", value: 9 },
{ id: "e", value: 5 },
@@ -3261,36 +3211,4 @@ describe("LSM merge insert", () => {
.execute([{ id: "g", value: 7 }]),
).rejects.toThrow();
});
it("auto-routes reads through the MemWAL scanner", async () => {
const conn = await connect(tmpDir.name);
const table = await bucketTable(conn); // base ids "a", "b"
await table
.mergeInsert("id")
.whenMatchedUpdateAll()
.whenNotMatchedInsertAll()
.execute([{ id: "c", value: 3 }]);
// Default read auto-routes and includes the active memtable row.
const lsm = await table.query().toArray();
expect(lsm.map((r) => r.id).sort()).toEqual(["a", "b", "c"]);
// useLsm(false) bypasses the MemWAL and reads the base table only.
const baseOnly = await table.query().useLsm(false).toArray();
expect(baseOnly.map((r) => r.id).sort()).toEqual(["a", "b"]);
});
it("reads the base table when no LSM spec is installed", async () => {
const conn = await connect(tmpDir.name);
const table = await conn.createEmptyTable(
"plain",
new arrow.Schema([new arrow.Field("id", new arrow.Utf8(), false)]),
);
// No spec: default read and useLsm(false) both succeed against the base table.
await expect(table.query().toArray()).resolves.toBeDefined();
await expect(table.query().useLsm(false).toArray()).resolves.toBeDefined();
// useLsm(true) demands MemWAL routing; without a spec it errors.
await expect(table.query().useLsm(true).toArray()).rejects.toThrow();
});
});
+1 -7
View File
@@ -29,14 +29,8 @@ test("full text search", async () => {
const tbl = await db.createTable("myVectors", data, { mode: "overwrite" });
await tbl.createIndex("doc", {
config: lancedb.Index.fts({
stem: false,
removeStopWords: true,
customStopWords: ["banana"],
}),
config: lancedb.Index.fts(),
});
const tokens = await tbl.tokenize("apple banana", { column: "doc" });
expect(tokens.map((token) => token.text)).toEqual(["apple"]);
// --8<-- [start:full_text_search]
const result = await tbl
-62
View File
@@ -1,7 +1,6 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { tableFromIPC } from "apache-arrow";
import {
Data,
SchemaLike,
@@ -21,9 +20,6 @@ import type {
CreateNamespaceResponse,
DescribeNamespaceResponse,
DropNamespaceResponse,
Job,
JobDescription,
JobInfo,
ListNamespacesResponse,
} from "./native";
export type {
@@ -440,40 +436,6 @@ export abstract class Connection {
newName: string,
options?: RenameTableOptions,
): Promise<void>;
/**
* A {@link Job} handle for a server-side job by id.
*
* The handle is constructed without a server round trip; an unknown id
* surfaces when the handle is used. Dropping the handle has no effect on
* the job itself.
*/
abstract job(jobId: string): Job;
/** List server-side jobs across the database's tables. */
abstract listJobs(): Promise<JobInfo[]>;
/**
* Describe a single server-side job by id.
*
* Resolves to `null` when the server has no such job.
*/
abstract getJob(jobId: string): Promise<JobDescription | null>;
/**
* Request cancellation of a server-side job by id.
*
* Resolves to true if the server accepted the cancellation, false if no
* such job exists. Cancelling an already-terminal job is a no-op success.
*/
abstract cancelJob(jobId: string): Promise<boolean>;
/**
* The lifecycle event history of a server-side job, as an Arrow table.
*
* Lists history across all jobs when `jobId` is omitted.
*/
abstract jobHistory(jobId?: string): Promise<ArrowTable>;
}
/** @hideconstructor */
@@ -760,30 +722,6 @@ export class LocalConnection extends Connection {
options?.newNamespacePath,
);
}
job(jobId: string): Job {
return this.inner.job(jobId);
}
async listJobs(): Promise<JobInfo[]> {
return this.inner.listJobs();
}
async getJob(jobId: string): Promise<JobDescription | null> {
return this.inner.getJob(jobId);
}
async cancelJob(jobId: string): Promise<boolean> {
return this.inner.cancelJob(jobId);
}
async jobHistory(jobId?: string): Promise<ArrowTable> {
const buf = await this.inner.jobHistory(jobId);
if (buf.length === 0) {
return new ArrowTable();
}
return tableFromIPC(buf);
}
}
/**
+1 -18
View File
@@ -85,13 +85,7 @@ export {
RenameTableOptions,
} from "./connection";
export {
Job,
JobDescription,
JobFailureInfo,
JobInfo,
Session,
} from "./native.js";
export { Session } from "./native.js";
export {
ExecutableQuery,
@@ -200,16 +194,6 @@ export interface TokenizeOptions {
/** Whether to remove stop words. */
removeStopWords?: boolean;
/**
* Custom stop words that replace the built-in list for `language`.
*
* This option only affects tokenization when `removeStopWords` is true.
*
* `undefined` keeps the built-in language list. An empty array explicitly
* replaces it with no stop words.
*/
customStopWords?: string[];
/** Whether to fold ASCII characters. */
asciiFolding?: boolean;
@@ -241,7 +225,6 @@ export async function tokenize(
options?.lowercase,
options?.stem,
options?.removeStopWords,
options?.customStopWords,
options?.asciiFolding,
options?.ngramMinLength,
options?.ngramMaxLength,
-20
View File
@@ -553,16 +553,6 @@ export interface FtsOptions {
*/
removeStopWords?: boolean;
/**
* Custom stop words that replace the built-in list for `language`.
*
* This option only affects tokenization when `removeStopWords` is true.
*
* `undefined` keeps the built-in language list. An empty array explicitly
* replaces it with no stop words.
*/
customStopWords?: string[];
/**
* whether to remove punctuation
*/
@@ -582,14 +572,6 @@ export interface FtsOptions {
* whether to only index the prefix of the token for ngram tokenizer
*/
prefixOnly?: boolean;
/**
* Number of documents per compressed posting block.
*
* The default is 128. Supported values are 128 and 256. A value of 256 uses
* the experimental FTS V3 format and may introduce breaking changes.
*/
blockSize?: 128 | 256;
}
export class Index {
@@ -765,12 +747,10 @@ export class Index {
options?.lowercase,
options?.stem,
options?.removeStopWords,
options?.customStopWords,
options?.asciiFolding,
options?.ngramMinLength,
options?.ngramMaxLength,
options?.prefixOnly,
options?.blockSize,
),
);
}
+11 -7
View File
@@ -88,17 +88,21 @@ export class MergeInsertBuilder {
);
}
/**
* Control MemWAL routing for this merge.
* Controls whether the merge uses the MemWAL LSM write path.
*
* By default (unset), a `mergeInsert` on a table with an LSM write spec is
* routed through Lance's MemWAL shard writer, and a table without one uses the
* standard path.
* routed through Lance's MemWAL shard writer, and a table without one uses
* the standard path. Pass `false` to force the standard path even when a
* spec is set. Pass `true` to require a spec — `mergeInsert` rejects if none
* is installed.
*
* @param enable - `true` forces MemWAL routing and errors if the table has no
* LSM write spec. `false` forces the standard write path even when a spec is set.
* @param useLsmWrite - Whether to use the LSM write path.
*/
useLsm(enable: boolean): MergeInsertBuilder {
return new MergeInsertBuilder(this.#native.useLsm(enable), this.#schema);
useLsmWrite(useLsmWrite: boolean): MergeInsertBuilder {
return new MergeInsertBuilder(
this.#native.useLsmWrite(useLsmWrite),
this.#schema,
);
}
/**
* Controls how an LSM merge checks that its input targets a single shard.
-38
View File
@@ -460,30 +460,6 @@ export class StandardQueryBase<
this.doCall((inner: NativeQueryType) => inner.fastSearch());
return this;
}
/**
* Control MemWAL read routing for this query.
*
* By default (unset), when the table carries a MemWAL write spec (see
* {@link Table#setLsmWriteSpec}), reads are routed through the LSM scanner so
* they also return data written via the `mergeInsert` LSM path that has not yet
* been compacted into the base table (the active/frozen in-memory memtables and
* the flushed generations), deduplicated by primary key; a table without a spec
* reads the base table.
*
* @param enable - `true` forces the LSM scanner and errors if the table has no
* MemWAL write spec. `false` bypasses the MemWAL and reads the base table only,
* even when a spec is present.
*
* Note: the LSM scanner does not support every query shape (e.g. reranking,
* hybrid search, `orderBy`). On a MemWAL table those shapes error unless
* `useLsm(false)` is set, because a base-only read would silently exclude
* un-compacted MemWAL data.
*/
useLsm(enable: boolean): this {
this.doCall((inner: NativeQueryType) => inner.useLsm(enable));
return this;
}
}
/**
@@ -772,20 +748,6 @@ export class TakeQuery extends QueryBase<NativeTakeQuery> {
constructor(inner: NativeTakeQuery) {
super(inner);
}
/**
* Control MemWAL read routing for this take query.
*
* `false` bypasses the MemWAL and reads the base table only — the escape hatch,
* since take-by-row-id/offset is not supported on the LSM scanner and, on a
* MemWAL table, auto-routes to it and errors otherwise.
*
* @param enable - `false` reads the base table only.
*/
useLsm(enable: boolean): this {
this.doCall((inner: NativeTakeQuery) => inner.useLsm(enable));
return this;
}
}
/** A builder for LanceDB queries.
+1 -1
View File
@@ -84,7 +84,7 @@ export function sanitizeMetadata(
throw Error("Expected metadata, if present, to be a Map<string, string>");
}
for (const item of metadataLike) {
if (typeof item[0] !== "string" || typeof item[1] !== "string") {
if (!(typeof item[0] === "string" || !(typeof item[1] === "string"))) {
throw Error(
"Expected metadata, if present, to be a Map<string, string> but it had non-string keys or values",
);
-28
View File
@@ -30,7 +30,6 @@ import {
DropColumnsResult,
IndexConfig,
IndexStatistics,
Job,
Branches as NativeBranches,
OptimizeStats,
TableStatistics,
@@ -359,17 +358,6 @@ export abstract class Table {
options?: Partial<IndexOptions>,
): Promise<void>;
/**
* Create an index, returning a handle to the indexing job.
*
* The job may already be complete when returned; callers must not assume
* the index exists until {@link Job.wait} resolves.
*/
abstract createIndexAsync(
column: string,
options?: Partial<IndexOptions>,
): Promise<Job>;
/**
* Drop an index from the table.
*
@@ -952,22 +940,6 @@ export class LocalTable extends Table {
);
}
async createIndexAsync(
column: string,
options?: Partial<IndexOptions>,
): Promise<Job> {
// biome-ignore lint/suspicious/noExplicitAny: skip
const nativeIndex = (options?.config as any)?.inner;
return await this.inner.createIndexAsync(
nativeIndex,
column,
options?.replace,
options?.waitTimeoutSeconds,
options?.name,
options?.train,
);
}
async dropIndex(name: string): Promise<void> {
await this.inner.dropIndex(name);
}
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-darwin-arm64",
"version": "0.37.1-beta.0",
"version": "0.32.0-beta.2",
"os": ["darwin"],
"cpu": ["arm64"],
"main": "lancedb.darwin-arm64.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-gnu",
"version": "0.37.1-beta.0",
"version": "0.32.0-beta.2",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-gnu.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-musl",
"version": "0.37.1-beta.0",
"version": "0.32.0-beta.2",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-musl.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-gnu",
"version": "0.37.1-beta.0",
"version": "0.32.0-beta.2",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-gnu.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-musl",
"version": "0.37.1-beta.0",
"version": "0.32.0-beta.2",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-musl.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-arm64-msvc",
"version": "0.37.1-beta.0",
"version": "0.32.0-beta.2",
"os": [
"win32"
],
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-x64-msvc",
"version": "0.37.1-beta.0",
"version": "0.32.0-beta.2",
"os": ["win32"],
"cpu": ["x64"],
"main": "lancedb.win32-x64-msvc.node",
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "@lancedb/lancedb",
"version": "0.37.1-beta.0",
"version": "0.32.0-beta.2",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@lancedb/lancedb",
"version": "0.37.1-beta.0",
"version": "0.32.0-beta.2",
"cpu": [
"x64",
"arm64"
+1 -1
View File
@@ -11,7 +11,7 @@
"ann"
],
"private": false,
"version": "0.37.1-beta.0",
"version": "0.32.0-beta.2",
"main": "dist/index.js",
"exports": {
".": "./dist/index.js",
-63
View File
@@ -340,69 +340,6 @@ impl Connection {
self.get_inner()?.drop_all_tables(&ns).await.default_error()
}
/// A `Job` handle for a server-side job by id.
///
/// The handle is constructed without a server round trip; an unknown id
/// surfaces when the handle is used.
#[napi]
pub fn job(&self, job_id: String) -> napi::Result<crate::job::Job> {
let job = self.get_inner()?.job(job_id).default_error()?;
Ok(crate::job::Job::new(job))
}
/// List server-side jobs across the database's tables.
#[napi(catch_unwind)]
pub async fn list_jobs(&self) -> napi::Result<Vec<crate::job::JobInfo>> {
let jobs = self.get_inner()?.list_jobs().await.default_error()?;
Ok(jobs.into_iter().map(Into::into).collect())
}
/// Describe a single server-side job by id. `null` when the server has
/// no such job.
#[napi(catch_unwind)]
pub async fn get_job(
&self,
job_id: String,
) -> napi::Result<Option<crate::job::JobDescription>> {
let description = self.get_inner()?.get_job(&job_id).await.default_error()?;
Ok(description.map(Into::into))
}
/// Request cancellation of a server-side job by id. Returns true if the
/// server accepted the cancellation, false if no such job exists.
#[napi(catch_unwind)]
pub async fn cancel_job(&self, job_id: String) -> napi::Result<bool> {
self.get_inner()?.cancel_job(&job_id).await.default_error()
}
/// The lifecycle event history of a server-side job (all jobs when
/// `job_id` is null), as an Arrow IPC stream buffer. Empty when there is
/// no history.
#[napi(catch_unwind)]
pub async fn job_history(&self, job_id: Option<String>) -> napi::Result<Buffer> {
let batches = self
.get_inner()?
.job_history(job_id.as_deref())
.await
.default_error()?;
let Some(first) = batches.first() else {
return Ok(Buffer::from(Vec::<u8>::new()));
};
let mut out = Vec::new();
let mut writer = arrow_ipc::writer::StreamWriter::try_new(&mut out, &first.schema())
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
for batch in &batches {
writer
.write(batch)
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
}
writer
.finish()
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
drop(writer);
Ok(Buffer::from(out))
}
#[napi(catch_unwind)]
/// Describe a namespace and return its properties.
pub async fn describe_namespace(
+3 -13
View File
@@ -43,7 +43,6 @@ pub fn tokenize(
lower_case: Option<bool>,
stem: Option<bool>,
remove_stop_words: Option<bool>,
custom_stop_words: Option<Vec<String>>,
ascii_folding: Option<bool>,
ngram_min_length: Option<u32>,
ngram_max_length: Option<u32>,
@@ -73,7 +72,6 @@ pub fn tokenize(
if let Some(remove_stop_words) = remove_stop_words {
opts = opts.remove_stop_words(remove_stop_words);
}
opts = opts.custom_stop_words(custom_stop_words);
if let Some(ascii_folding) = ascii_folding {
opts = opts.ascii_folding(ascii_folding);
}
@@ -224,13 +222,11 @@ impl Index {
lower_case: Option<bool>,
stem: Option<bool>,
remove_stop_words: Option<bool>,
custom_stop_words: Option<Vec<String>>,
ascii_folding: Option<bool>,
ngram_min_length: Option<u32>,
ngram_max_length: Option<u32>,
prefix_only: Option<bool>,
block_size: Option<u32>,
) -> napi::Result<Self> {
) -> Self {
let mut opts = FtsIndexBuilder::default();
if let Some(with_position) = with_position {
opts = opts.with_position(with_position);
@@ -253,7 +249,6 @@ impl Index {
if let Some(remove_stop_words) = remove_stop_words {
opts = opts.remove_stop_words(remove_stop_words);
}
opts = opts.custom_stop_words(custom_stop_words);
if let Some(ascii_folding) = ascii_folding {
opts = opts.ascii_folding(ascii_folding);
}
@@ -266,15 +261,10 @@ impl Index {
if let Some(prefix_only) = prefix_only {
opts = opts.ngram_prefix_only(prefix_only);
}
if let Some(block_size) = block_size {
opts = opts
.block_size(block_size as usize)
.map_err(|err| napi::Error::from_reason(err.to_string()))?;
}
Ok(Self {
Self {
inner: Mutex::new(Some(LanceDbIndex::FTS(opts))),
})
}
}
#[napi(factory)]
-123
View File
@@ -1,123 +0,0 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::sync::Arc;
use napi_derive::napi;
use crate::error::NapiErrorExt;
/// A handle to an operation that may still be running.
#[napi]
pub struct Job {
inner: Arc<lancedb::Job>,
}
impl Job {
pub(crate) fn new(inner: lancedb::Job) -> Self {
Self {
inner: Arc::new(inner),
}
}
}
#[napi]
impl Job {
/// Identifies the operation on the server that is running it. Operations
/// that run in this process have no server id. The value is opaque.
#[napi(getter)]
pub fn id(&self) -> Option<String> {
self.inner.id().map(str::to_string)
}
/// The operation's current lifecycle state: "running", "finished",
/// "failed", or "cancelled".
///
/// A point snapshot; unlike {@link Job.wait} it does not block or reject
/// on a terminal failure state. States a newer server reports that this
/// client version does not know pass through as-is.
#[napi(catch_unwind)]
pub async fn status(&self) -> napi::Result<String> {
self.inner.status().await.default_error()
}
/// Wait until the operation reaches a terminal state.
#[napi(catch_unwind)]
pub async fn wait(&self) -> napi::Result<()> {
self.inner.wait().await.default_error()
}
/// Request cancellation. Cancelling a finished operation is a no-op.
#[napi(catch_unwind)]
pub async fn cancel(&self) -> napi::Result<()> {
self.inner.cancel().await.default_error()
}
}
/// A row from `Connection.listJobs`: one server-side job.
#[napi(object)]
pub struct JobInfo {
/// The job id -- what `Connection.getJob` and `Connection.cancelJob`
/// accept.
pub job_id: String,
/// The table the job runs against, without URI or namespace.
pub table: String,
pub job_type: String,
/// Lifecycle state: "running", "finished", "failed", or "cancelled".
pub state: String,
/// When the job was created, in milliseconds since the epoch.
pub created_at_millis: i64,
}
impl From<lancedb::database::JobInfo> for JobInfo {
fn from(info: lancedb::database::JobInfo) -> Self {
Self {
job_id: info.job_id,
table: info.table,
job_type: info.job_type,
state: info.state,
created_at_millis: info.created_at_millis,
}
}
}
/// The server's account of why a job failed.
#[napi(object)]
pub struct JobFailureInfo {
pub phase: Option<String>,
pub message: Option<String>,
pub retryable: Option<bool>,
}
/// A described job from `Connection.getJob`.
#[napi(object)]
pub struct JobDescription {
pub job_id: String,
pub job_type: String,
/// Lifecycle state: "running", "finished", "failed", or "cancelled".
pub state: String,
/// When the job was created, in milliseconds since the epoch.
pub creation_ms: i64,
/// The job-type-specific specification as a JSON string, when present.
pub spec_json: Option<String>,
/// Why the job failed, when the job is failed and the server reports a
/// reason.
pub failure: Option<JobFailureInfo>,
}
impl From<lancedb::database::JobDescription> for JobDescription {
fn from(description: lancedb::database::JobDescription) -> Self {
Self {
job_id: description.job_id,
job_type: description.job_type,
state: description.state,
creation_ms: description.creation_ms,
spec_json: (!description.spec.is_null()).then(|| description.spec.to_string()),
failure: description.failure.map(|failure| JobFailureInfo {
phase: failure.phase,
message: failure.message,
retryable: failure.retryable,
}),
}
}
}
-1
View File
@@ -11,7 +11,6 @@ mod error;
mod header;
mod index;
mod iterator;
mod job;
pub mod merge;
pub mod otel;
pub mod permutation;
+2 -2
View File
@@ -51,9 +51,9 @@ impl NativeMergeInsertBuilder {
}
#[napi]
pub fn use_lsm(&self, enable: bool) -> Self {
pub fn use_lsm_write(&self, use_lsm_write: bool) -> Self {
let mut this = self.clone();
this.inner.use_lsm(enable);
this.inner.use_lsm_write(use_lsm_write);
this
}
-15
View File
@@ -168,11 +168,6 @@ impl Query {
self.inner = self.inner.clone().with_row_id();
}
#[napi]
pub fn use_lsm(&mut self, enable: bool) {
self.inner = self.inner.clone().use_lsm(enable);
}
#[napi]
pub fn order_by(&mut self, ordering: Option<Vec<ColumnOrdering>>) -> napi::Result<()> {
let ordering = ordering.map(|ordering| {
@@ -379,11 +374,6 @@ impl VectorQuery {
self.inner = self.inner.clone().with_row_id();
}
#[napi]
pub fn use_lsm(&mut self, enable: bool) {
self.inner = self.inner.clone().use_lsm(enable);
}
#[napi]
pub fn rerank(
&mut self,
@@ -489,11 +479,6 @@ impl TakeQuery {
self.inner = self.inner.clone().with_row_id();
}
#[napi]
pub fn use_lsm(&mut self, enable: bool) {
self.inner = self.inner.clone().use_lsm(enable);
}
#[napi(catch_unwind)]
pub async fn output_schema(&self) -> napi::Result<Buffer> {
let schema = self.inner.output_schema().await.default_error()?;
+2 -39
View File
@@ -168,39 +168,6 @@ impl Table {
builder.execute().await.default_error()
}
#[napi(catch_unwind)]
pub async fn create_index_async(
&self,
index: Option<&Index>,
column: String,
replace: Option<bool>,
wait_timeout_s: Option<i64>,
name: Option<String>,
train: Option<bool>,
) -> napi::Result<crate::job::Job> {
let lancedb_index = if let Some(index) = index {
index.consume()?
} else {
lancedb::index::Index::Auto
};
let mut builder = self.inner_ref()?.create_index(&[column], lancedb_index);
if let Some(replace) = replace {
builder = builder.replace(replace);
}
if let Some(timeout) = wait_timeout_s {
builder =
builder.wait_timeout(std::time::Duration::from_secs(timeout.try_into().unwrap()));
}
if let Some(name) = name {
builder = builder.name(name);
}
if let Some(train) = train {
builder = builder.train(train);
}
let job = builder.execute_async().await.default_error()?;
Ok(crate::job::Job::new(job))
}
#[napi(catch_unwind)]
pub async fn drop_index(&self, index_name: String) -> napi::Result<()> {
self.inner_ref()?
@@ -339,9 +306,7 @@ impl Table {
let transforms = NewColumnTransform::SqlExpressions(transforms);
let res = self
.inner_ref()?
.add_columns()
.transform(transforms)
.execute()
.add_columns(transforms, None)
.await
.default_error()?;
Ok(res.into())
@@ -358,9 +323,7 @@ impl Table {
let transforms = NewColumnTransform::AllNulls(schema);
let res = self
.inner_ref()?
.add_columns()
.transform(transforms)
.execute()
.add_columns(transforms, None)
.await
.default_error()?;
Ok(res.into())
+49
View File
@@ -0,0 +1,49 @@
[tool.bumpversion]
current_version = "0.35.0-beta.2"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.
(?P<patch>0|[1-9]\\d*)
(?:-(?P<pre_l>[a-zA-Z-]+)\\.(?P<pre_n>0|[1-9]\\d*))?
"""
serialize = [
"{major}.{minor}.{patch}-{pre_l}.{pre_n}",
"{major}.{minor}.{patch}",
]
search = "{current_version}"
replace = "{new_version}"
regex = false
ignore_missing_version = false
ignore_missing_files = false
tag = true
sign_tags = false
tag_name = "python-v{new_version}"
tag_message = "Bump version: {current_version} → {new_version}"
allow_dirty = true
commit = true
message = "Bump version: {current_version} → {new_version}"
commit_args = ""
# bump-my-version >=1.4.0 rejects pre_commit_hooks containing shell syntax unless opted in.
allow_shell_hooks = true
# Update Cargo.lock after version bump
pre_commit_hooks = [
"""
cd python && cargo update -p lancedb-python
if git diff --quiet ../Cargo.lock; then
echo "Cargo.lock unchanged"
else
git add ../Cargo.lock
echo "Updated and staged Cargo.lock"
fi
""",
]
[tool.bumpversion.parts.pre_l]
values = ["beta", "final"]
optional_value = "final"
[[tool.bumpversion.files]]
filename = "Cargo.toml"
search = "\nversion = \"{current_version}\""
replace = "\nversion = \"{new_version}\""
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb-python"
version = "0.37.1-beta.0"
version = "0.35.0-beta.2"
publish = false
edition.workspace = true
description = "Python bindings for LanceDB"
+2 -22
View File
@@ -20,7 +20,6 @@ from .remote import ClientConfig
from .remote.db import RemoteDBConnection
from .expr import Expr, col, lit, func
from .schema import blob, vector, BlobType
from .job import AsyncJob, Job
from .table import AsyncTable, Table
from .types import BaseTokenizerType
from ._lancedb import Session
@@ -162,15 +161,6 @@ def connect(
... },
... )
Azure managed identity authentication requires only the storage account name.
LanceDB acquires and refreshes managed identity tokens automatically. For a
user-assigned identity, also set ``azure_storage_client_id``:
>>> db = lancedb.connect( # doctest: +SKIP
... "az://my-container/lancedb",
... storage_options={"account_name": "my-storage-account"},
... )
For tests and temporary data, use an in-memory database:
>>> db = lancedb.connect("memory://")
@@ -268,7 +258,6 @@ def tokenize(
lower_case: bool = True,
stem: bool = True,
remove_stop_words: bool = True,
custom_stop_words: Optional[List[str]] = None,
ascii_folding: bool = True,
ngram_min_length: int = 3,
ngram_max_length: int = 3,
@@ -276,10 +265,9 @@ def tokenize(
) -> Iterable[FtsToken]:
"""Tokenize a full-text search query using an explicit tokenizer.
This does not require an FTS index. The tokenizer options match
:class:`lancedb.index.FTS`. ``custom_stop_words`` accepts a list of strings.
This does not require a table or FTS index. The tokenizer options match
:class:`lancedb.index.FTS`.
"""
return _tokenize(
query,
base_tokenizer=base_tokenizer,
@@ -288,7 +276,6 @@ def tokenize(
lower_case=lower_case,
stem=stem,
remove_stop_words=remove_stop_words,
custom_stop_words=custom_stop_words,
ascii_folding=ascii_folding,
ngram_min_length=ngram_min_length,
ngram_max_length=ngram_max_length,
@@ -464,11 +451,6 @@ async def connect_async(
... db = await lancedb.connect_async("s3://my-bucket/lancedb",
... storage_options={
... "aws_access_key_id": "***"})
... # Azure managed identity tokens are acquired and refreshed automatically
... db = await lancedb.connect_async(
... "az://my-container/lancedb",
... storage_options={"account_name": "my-storage-account"},
... )
... # For tests and temporary data, use an in-memory database
... db = await lancedb.connect_async("memory://")
... # Connect to LanceDB cloud
@@ -515,7 +497,6 @@ __all__ = [
"connect_namespace",
"connect_namespace_async",
"AsyncConnection",
"AsyncJob",
"AsyncLanceNamespaceDBConnection",
"AsyncTable",
"FtsToken",
@@ -529,7 +510,6 @@ __all__ = [
"BlobType",
"vector",
"DBConnection",
"Job",
"LanceDBConnection",
"LanceNamespaceDBConnection",
"RemoteDBConnection",
+23 -6
View File
@@ -14,10 +14,14 @@ import pyarrow as pa
from .expr import Expr
from .schema import blob_v2_column_paths
from .types import BlobMode, QueryProjection, QueryProjectionSpec
from .util import get_uri_scheme
if TYPE_CHECKING:
from _typeshed import WriteableBuffer
from .remote.table import RemoteTable
from .table import AsyncTable, Table
BLOB_MODE_TO_HANDLING = {
"lazy": "blobs_descriptions",
"bytes": "all_binary",
@@ -100,6 +104,22 @@ def validate_blob_mode(blob_mode: BlobMode) -> None:
raise ValueError(f"blob_mode must be one of {modes}, got {blob_mode!r}")
def supports_blob_auto_row_id(table: Table | AsyncTable | RemoteTable) -> bool:
"""Blob auto row-id applies to native tables, not LanceDB Cloud."""
from .remote.table import RemoteTable
if isinstance(table, RemoteTable):
return False
inner = getattr(table, "_inner", None)
if inner is not None:
uri = inner.database().uri
if isinstance(uri, str) and get_uri_scheme(uri) == "db":
return False
return True
def projection_includes_blob_column(
projection: QueryProjection,
blob_columns: Iterable[str],
@@ -144,14 +164,16 @@ def v2_projection_needs_row_id(
def blob_auto_row_id_for_scan(
table: Table | AsyncTable | RemoteTable,
schema: pa.Schema,
projection: QueryProjection,
*,
with_row_id: bool | None,
) -> bool:
"""Auto row-id only applies when the caller said nothing about row ids."""
if with_row_id is not None:
return False
if not supports_blob_auto_row_id(table):
return False
return v2_projection_needs_row_id(schema, projection, with_row_id=False)
@@ -164,11 +186,6 @@ def finalize_blob_query_table(
) -> pa.Table:
if user_requested_row_id or not blob_auto_row_id:
return tbl
if "_rowid" not in tbl.column_names:
# A backend that ignores the row-id request leaves nothing to stash. Hand
# back the projection as-is so fetch_blobs raises the error that names the
# ways to supply row ids, rather than failing here about a hidden column.
return tbl
return stash_auto_row_ids(tbl, blob_paths)
-82
View File
@@ -59,7 +59,6 @@ def tokenize(
lower_case: bool = True,
stem: bool = True,
remove_stop_words: bool = True,
custom_stop_words: Optional[List[str]] = None,
ascii_folding: bool = True,
ngram_min_length: int = 3,
ngram_max_length: int = 3,
@@ -146,13 +145,6 @@ class Connection(object):
start_after: Optional[str],
limit: Optional[int],
) -> list[str]: ... # Deprecated: Use list_tables instead
def job(self, job_id: str) -> Job: ...
async def list_jobs(self) -> List[JobInfo]: ...
async def get_job(self, job_id: str) -> Optional[JobDescription]: ...
async def cancel_job(self, job_id: str) -> bool: ...
async def job_history(
self, job_id: Optional[str] = None
) -> List[pa.RecordBatch]: ...
async def create_table(
self,
name: str,
@@ -216,47 +208,6 @@ class BlobFile:
def read_range(self, offset: int, length: int) -> bytes: ...
def read_up_to(self, length: int) -> bytes: ...
class Job:
@property
def id(self) -> Optional[str]: ...
async def status(self) -> str: ...
async def wait(self) -> None: ...
async def cancel(self) -> None: ...
class JobInfo:
@property
def job_id(self) -> str: ...
@property
def table(self) -> str: ...
@property
def job_type(self) -> str: ...
@property
def state(self) -> str: ...
@property
def created_at_millis(self) -> int: ...
class JobFailureInfo:
@property
def phase(self) -> Optional[str]: ...
@property
def message(self) -> Optional[str]: ...
@property
def retryable(self) -> Optional[bool]: ...
class JobDescription:
@property
def job_id(self) -> str: ...
@property
def job_type(self) -> str: ...
@property
def state(self) -> str: ...
@property
def creation_ms(self) -> int: ...
@property
def spec_json(self) -> Optional[str]: ...
@property
def failure(self) -> Optional[JobFailureInfo]: ...
class Table:
def name(self) -> str: ...
def __repr__(self) -> str: ...
@@ -296,28 +247,6 @@ class Table:
name: Optional[str],
train: Optional[bool],
): ...
async def create_index_async(
self,
column: str,
index: Union[
IvfFlat,
IvfSq,
IvfPq,
HnswPq,
HnswSq,
HnswFlat,
BTree,
Bitmap,
LabelList,
Fm,
FTS,
],
replace: Optional[bool],
wait_timeout: Optional[object],
*,
name: Optional[str],
train: Optional[bool],
) -> Job: ...
async def list_versions(self) -> List[Dict[str, Any]]: ...
async def version(self) -> int: ...
async def checkout(self, version: Union[int, str]): ...
@@ -368,11 +297,6 @@ class Table:
async def fetch_blobs(
self, column: str, row_ids: list[int]
) -> pa.LargeBinaryArray: ...
async def fetch_blob_ranges(
self,
column: str,
requests: List[Tuple[int, int, int]],
) -> pa.LargeBinaryArray: ...
async def fetch_blob_files(
self, column: str, row_ids: list[int]
) -> list[Optional[BlobFile]]: ...
@@ -467,7 +391,6 @@ class Query:
def fast_search(self): ...
def with_row_id(self): ...
def postfilter(self): ...
def use_lsm(self, enable: bool): ...
def nearest_to(self, query_vec: pa.Array) -> VectorQuery: ...
def nearest_to_text(self, query: dict) -> FTSQuery: ...
def order_by(self, ordering: Optional[List[ColumnOrdering]]): ...
@@ -484,7 +407,6 @@ class Query:
class TakeQuery:
def select(self, columns: List[str]): ...
def with_row_id(self): ...
def use_lsm(self, enable: bool): ...
async def output_schema(self) -> pa.Schema: ...
async def execute(self) -> RecordBatchStream: ...
async def explain_plan(self, verbose: Optional[bool]) -> str: ...
@@ -503,7 +425,6 @@ class FTSQuery:
def fast_search(self): ...
def with_row_id(self): ...
def postfilter(self): ...
def use_lsm(self, enable: bool): ...
def get_query(self) -> str: ...
def add_query_vector(self, query_vec: pa.Array) -> None: ...
def nearest_to(self, query_vec: pa.Array) -> HybridQuery: ...
@@ -531,7 +452,6 @@ class VectorQuery:
def column(self, column: str): ...
def distance_type(self, distance_type: str): ...
def postfilter(self): ...
def use_lsm(self, enable: bool): ...
def refine_factor(self, refine_factor: int): ...
def nprobes(self, nprobes: int): ...
def minimum_nprobes(self, minimum_nprobes: int): ...
@@ -555,7 +475,6 @@ class HybridQuery:
def fast_search(self): ...
def with_row_id(self): ...
def postfilter(self): ...
def use_lsm(self, enable: bool): ...
def distance_type(self, distance_type: str): ...
def refine_factor(self, refine_factor: int): ...
def nprobes(self, nprobes: int): ...
@@ -580,7 +499,6 @@ class PyQueryRequest:
select: Optional[Union[str, List[str]]]
fast_search: Optional[bool]
with_row_id: Optional[bool]
use_lsm: Optional[bool]
column: Optional[str]
query_vector: Optional[List[pa.Array]]
minimum_nprobes: Optional[int]
+10 -185
View File
@@ -45,7 +45,6 @@ from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError
from . import __version__
from ._lancedb import connect as lancedb_connect # type: ignore
from .job import AsyncJob, Job
from .table import (
AsyncTable,
LanceTable,
@@ -64,7 +63,6 @@ if TYPE_CHECKING:
from .pydantic import LanceModel
from ._lancedb import Connection as LanceDbConnection
from ._lancedb import JobDescription, JobInfo
from .common import DATA, URI
from .embeddings import EmbeddingFunctionConfig
from ._lancedb import Session
@@ -180,51 +178,6 @@ class DBConnection(EnforceOverrides):
"Namespace operations are not supported for this connection type"
)
def namespace_exists(self, namespace_id: List[str]) -> bool:
"""Check if a namespace exists.
Parameters
----------
namespace_id: List[str]
The namespace identifier to check.
Returns
-------
bool
True if the namespace exists, False otherwise.
Raises
------
NotImplementedError
If the connection type does not support namespace operations.
"""
raise NotImplementedError(
"Namespace operations are not supported for this connection type"
)
def table_exists(self, table_id: List[str]) -> bool:
"""Check if a table exists.
Parameters
----------
table_id: List[str]
The table identifier to check (full path including namespace
segments and table name).
Returns
-------
bool
True if the table exists, False otherwise.
Raises
------
NotImplementedError
If the connection type does not support namespace operations.
"""
raise NotImplementedError(
"Namespace operations are not supported for this connection type"
)
def list_tables(
self,
namespace_path: Optional[List[str]] = None,
@@ -406,7 +359,7 @@ class DBConnection(EnforceOverrides):
Data is converted to Arrow before being written to disk. For maximum
control over how data is saved, either provide the PyArrow schema to
convert to or else provide a [PyArrow Table][pyarrow.Table] directly.
convert to or else provide a [PyArrow Table](pyarrow.Table) directly.
>>> import pyarrow as pa
>>> custom_schema = pa.schema([
@@ -610,46 +563,6 @@ class DBConnection(EnforceOverrides):
"""
raise NotImplementedError("serialize is not supported for this connection type")
def job(self, job_id: str) -> Job:
"""A [Job][lancedb.job.Job] handle for a server-side job by id.
The handle is constructed without a server round trip; an unknown id
surfaces when the handle is used. Dropping the handle has no effect
on the job itself.
"""
raise NotImplementedError("job is not supported for this connection type")
def list_jobs(self) -> List[JobInfo]:
"""List server-side jobs across the database's tables."""
raise NotImplementedError("list_jobs is not supported for this connection type")
def get_job(self, job_id: str) -> Optional[JobDescription]:
"""Describe a single server-side job by id.
Returns None when the server has no such job.
"""
raise NotImplementedError("get_job is not supported for this connection type")
def cancel_job(self, job_id: str) -> bool:
"""Request cancellation of a server-side job by id.
Returns True if the server accepted the cancellation, False if no
such job exists. Cancelling an already-terminal job is a no-op
success.
"""
raise NotImplementedError(
"cancel_job is not supported for this connection type"
)
def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
"""The lifecycle event history of a server-side job, as Arrow batches.
Lists history across all jobs when `job_id` is None.
"""
raise NotImplementedError(
"job_history is not supported for this connection type"
)
class LanceDBConnection(DBConnection):
"""
@@ -707,9 +620,6 @@ class LanceDBConnection(DBConnection):
self._namespace_client_properties = namespace_client_properties
if _inner is not None:
self._conn = _inner
# Native-derived wrappers resolve this in their async reconstruction
# path so construction never synchronously re-enters LOOP.
self._read_consistency_interval = read_consistency_interval
self._cached_namespace_client = None
return
@@ -759,14 +669,11 @@ class LanceDBConnection(DBConnection):
# storage_options. Also, this class really shouldn't be holding any state
# beyond _conn.
self._conn = AsyncConnection(LOOP.run(do_connect()))
# Keep property access synchronous so debugger introspection cannot wait on
# the background loop while that thread is suspended at a breakpoint.
self._read_consistency_interval = read_consistency_interval
self._cached_namespace_client: Optional[LanceNamespace] = None
@property
def read_consistency_interval(self) -> Optional[timedelta]:
return self._read_consistency_interval
return LOOP.run(self._conn.get_read_consistency_interval())
@property
def session(self) -> Optional[Session]:
@@ -777,19 +684,15 @@ class LanceDBConnection(DBConnection):
return self._conn.uri
@classmethod
def from_inner(
cls,
inner: LanceDbConnection,
read_consistency_interval: Optional[timedelta],
):
return cls(
None,
read_consistency_interval=read_consistency_interval,
_inner=inner,
)
def from_inner(cls, inner: LanceDbConnection):
return cls(None, _inner=inner)
def __repr__(self) -> str:
return f"{self.__class__.__name__}(uri={self._conn.uri!r})"
val = f"{self.__class__.__name__}(uri={self._conn.uri!r}"
if self.read_consistency_interval is not None:
val += f", read_consistency_interval={repr(self.read_consistency_interval)}"
val += ")"
return val
@override
def serialize(self) -> str:
@@ -1226,47 +1129,6 @@ class LanceDBConnection(DBConnection):
)
)
@override
def job(self, job_id: str) -> Job:
"""A [Job][lancedb.job.Job] handle for a server-side job by id.
The handle is constructed without a server round trip; an unknown id
surfaces when the handle is used. Dropping the handle has no effect
on the job itself.
"""
return Job(self._conn.job(job_id))
@override
def list_jobs(self) -> List[JobInfo]:
"""List server-side jobs across the database's tables."""
return LOOP.run(self._conn.list_jobs())
@override
def get_job(self, job_id: str) -> Optional[JobDescription]:
"""Describe a single server-side job by id.
Returns None when the server has no such job.
"""
return LOOP.run(self._conn.get_job(job_id))
@override
def cancel_job(self, job_id: str) -> bool:
"""Request cancellation of a server-side job by id.
Returns True if the server accepted the cancellation, False if no
such job exists. Cancelling an already-terminal job is a no-op
success.
"""
return LOOP.run(self._conn.cancel_job(job_id))
@override
def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
"""The lifecycle event history of a server-side job, as Arrow batches.
Lists history across all jobs when `job_id` is None.
"""
return LOOP.run(self._conn.job_history(job_id))
@override
def namespace_client(self) -> LanceNamespace:
"""Get the equivalent namespace client for this connection.
@@ -1667,7 +1529,7 @@ class AsyncConnection(object):
Data is converted to Arrow before being written to disk. For maximum
control over how data is saved, either provide the PyArrow schema to
convert to or else provide a [PyArrow Table][pyarrow.Table] directly.
convert to or else provide a [PyArrow Table](pyarrow.Table) directly.
>>> import pyarrow as pa
>>> custom_schema = pa.schema([
@@ -1976,43 +1838,6 @@ class AsyncConnection(object):
namespace_path = []
await self._inner.drop_all_tables(namespace_path=namespace_path)
def job(self, job_id: str) -> AsyncJob:
"""An [AsyncJob][lancedb.job.AsyncJob] handle for a server-side job
by id.
The handle is constructed without a server round trip; an unknown id
surfaces when the handle is used. Dropping the handle has no effect
on the job itself.
"""
return AsyncJob(self._inner.job(job_id))
async def list_jobs(self) -> List[JobInfo]:
"""List server-side jobs across the database's tables."""
return await self._inner.list_jobs()
async def get_job(self, job_id: str) -> Optional[JobDescription]:
"""Describe a single server-side job by id.
Returns None when the server has no such job.
"""
return await self._inner.get_job(job_id)
async def cancel_job(self, job_id: str) -> bool:
"""Request cancellation of a server-side job by id.
Returns True if the server accepted the cancellation, False if no
such job exists. Cancelling an already-terminal job is a no-op
success.
"""
return await self._inner.cancel_job(job_id)
async def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
"""The lifecycle event history of a server-side job, as Arrow batches.
Lists history across all jobs when `job_id` is None.
"""
return await self._inner.job_history(job_id)
async def namespace_client(self) -> LanceNamespace:
"""Get the equivalent namespace client for this connection.
@@ -21,32 +21,3 @@ from .watsonx import WatsonxEmbeddings
from .voyageai import VoyageAIEmbeddingFunction
from .colpali import ColPaliEmbeddings
from .siglip import SigLipEmbeddings
# The API reference renders this package with a single mkdocstrings directive,
# which only picks up names listed here. New embedding functions must be added
# to both the imports above and this list, or they will silently go undocumented.
__all__ = [
"EmbeddingFunction",
"EmbeddingFunctionConfig",
"TextEmbeddingFunction",
"EmbeddingFunctionRegistry",
"get_registry",
"register",
"SentenceTransformerEmbeddings",
"OpenAIEmbeddings",
"OpenClipEmbeddings",
"BedRockText",
"CohereEmbeddingFunction",
"GeminiText",
"GteEmbeddings",
"InstructorEmbeddingFunction",
"JinaEmbeddings",
"OllamaEmbeddings",
"TransformersEmbeddingFunction",
"ColbertEmbeddings",
"VoyageAIEmbeddingFunction",
"WatsonxEmbeddings",
"ColPaliEmbeddings",
"ImageBindEmbeddings",
"SigLipEmbeddings",
]
+5 -5
View File
@@ -21,20 +21,20 @@ class BedRockText(TextEmbeddingFunction):
"""
Parameters
----------
name : str, default "amazon.titan-embed-text-v1"
name: str, default "amazon.titan-embed-text-v1"
The model ID of the bedrock model to use. Supported models for are:
- amazon.titan-embed-text-v1
- cohere.embed-english-v3
- cohere.embed-multilingual-v3
region : str, default "us-east-1"
region: str, default "us-east-1"
Optional name of the AWS Region in which the service should be called.
profile_name : str, default None
profile_name: str, default None
Optional name of the AWS profile to use for calling the Bedrock service.
If not specified, the default profile will be used.
assumed_role : str, default None
assumed_role: str, default None
Optional ARN of an AWS IAM role to assume for calling the Bedrock service.
If not specified, the current active credentials will be used.
role_session_name : str, default "lancedb-embeddings"
role_session_name: str, default "lancedb-embeddings"
Optional name of the AWS IAM role session to use for calling the Bedrock
service. If not specified, "lancedb-embeddings" name will be used.
+3 -5
View File
@@ -22,7 +22,7 @@ class CohereEmbeddingFunction(TextEmbeddingFunction):
Parameters
----------
name : str, default "embed-multilingual-v2.0"
name: str, default "embed-multilingual-v2.0"
The name of the model to use. List of acceptable models:
* embed-english-v3.0
@@ -33,14 +33,12 @@ class CohereEmbeddingFunction(TextEmbeddingFunction):
* embed-english-light-v2.0
* embed-multilingual-v2.0
source_input_type : str, default "search_document"
source_input_type: str, default "search_document"
The input type for the source column in the database
query_input_type : str, default "search_query"
query_input_type: str, default "search_query"
The input type for the query column in the database
Notes
-----
Cohere supports following input types:
| Input Type | Description |
+2 -2
View File
@@ -44,7 +44,7 @@ class ColPaliEmbeddings(EmbeddingFunction):
The token pooling strategy to use, by default "hierarchical".
- "hierarchical": Progressively pools tokens to reduce sequence length.
- "lambda": A simpler pooling that uses a custom `pooling_func`.
pooling_func : typing.Callable, optional
pooling_func: typing.Callable, optional
A function to use for pooling when `pooling_strategy` is "lambda".
pool_factor : int
Factor to reduce sequence length if token pooling is enabled (default 2).
@@ -52,7 +52,7 @@ class ColPaliEmbeddings(EmbeddingFunction):
Quantization configuration for the model. (default None, bitsandbytes needed)
batch_size : int
Batch size for processing inputs (default 2).
offload_folder : str, optional
offload_folder: str, optional
Folder to offload model weights if using CPU offloading (default None). This is
useful for large models that do not fit in memory.
"""
@@ -48,16 +48,16 @@ class GeminiText(TextEmbeddingFunction):
Parameters
----------
name : str, default "gemini-embedding-001"
name: str, default "gemini-embedding-001"
The name of the model to use. Supported models include:
- "gemini-embedding-001" (768 dimensions)
Note: The legacy "models/embedding-001" format is also supported but
"gemini-embedding-001" is recommended.
query_task_type : str, default "retrieval_query"
query_task_type: str, default "retrieval_query"
Sets the task type for the queries.
source_task_type : str, default "retrieval_document"
source_task_type: str, default "retrieval_document"
Sets the task type for ingestion.
Examples
+4 -4
View File
@@ -26,13 +26,13 @@ class GteEmbeddings(TextEmbeddingFunction):
Parameters
----------
name : str, default "thenlper/gte-large"
name: str, default "thenlper/gte-large"
The name of the model to use.
device : str, default "cpu"
device: str, default "cpu"
Sets the device type for the model.
normalize : str, default "True"
normalize: str, default "True"
Controls normalize param in encode function for the transformer.
mlx : bool, default False
mlx: bool, default False
Controls which model to use. False for gte-large,True for the mlx version.
Examples
@@ -35,23 +35,23 @@ class InstructorEmbeddingFunction(TextEmbeddingFunction):
Parameters
----------
name : str
name: str
The name of the model to use. Available models are listed at
https://github.com/xlang-ai/instructor-embedding#model-list;
The default model is hkunlp/instructor-base
batch_size : int, default 32
batch_size: int, default 32
The batch size to use when generating embeddings
device : str, default "cpu"
device: str, default "cpu"
The device to use when generating embeddings
show_progress_bar : bool, default True
show_progress_bar: bool, default True
Whether to show a progress bar when generating embeddings
normalize_embeddings : bool, default True
normalize_embeddings: bool, default True
Whether to normalize the embeddings
quantize : bool, default False
quantize: bool, default False
Whether to quantize the model
source_instruction : str, default "represent the document for retrieval"
source_instruction: str, default "represent the document for retrieval"
The instruction for the source column
query_instruction : str, default "represent the document for retrieving the most
query_instruction: str, default "represent the document for retrieving the most
similar documents"
The instruction for the query
+2 -2
View File
@@ -40,10 +40,10 @@ class JinaEmbeddings(EmbeddingFunction):
Parameters
----------
name : str, default "jina-clip-v1". Note that some models support both image
name: str, default "jina-clip-v1". Note that some models support both image
and text embeddings and some just text embedding
api_key : str, default None
api_key: str, default None
The api key to access Jina API. If you pass None, you can set JINA_API_KEY
environment variable
@@ -21,13 +21,13 @@ class SentenceTransformerEmbeddings(TextEmbeddingFunction):
Parameters
----------
name : str, default "all-MiniLM-L6-v2"
name: str, default "all-MiniLM-L6-v2"
The name of the model to use.
device : str, default "cpu"
device: str, default "cpu"
The device to use for the model
normalize : bool, default True
normalize: bool, default True
Whether to normalize the embeddings
trust_remote_code : bool, default True
trust_remote_code: bool, default True
Whether to trust the remote code
"""
+2 -2
View File
@@ -167,7 +167,7 @@ class VoyageAIEmbeddingFunction(EmbeddingFunction):
Parameters
----------
name : str
name: str
The name of the model to use. List of acceptable models:
* voyage-4 (1024 dims, general-purpose and multilingual retrieval)
@@ -185,7 +185,7 @@ class VoyageAIEmbeddingFunction(EmbeddingFunction):
* voyage-law-2
* voyage-code-2
output_dimension : int, optional
output_dimension: int, optional
The output dimension for models that support flexible dimensions.
Currently only voyage-multimodal-3.5 supports this feature.
Valid options: 256, 512, 1024 (default), 2048.
-12
View File
@@ -23,15 +23,3 @@ class MissingColumnError(KeyError):
return (
f"Error: Column '{self.column_name}' does not exist in the DataFrame object"
)
class JobFailedError(RuntimeError):
"""Exception raised when an asynchronous job reaches the failed state."""
pass
class JobCancelledError(RuntimeError):
"""Exception raised when an asynchronous job was cancelled."""
pass
+24 -44
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
from dataclasses import dataclass
from typing import List, Literal, Optional
from typing import Literal, Optional
from ._lancedb import (
IndexConfig,
@@ -115,12 +115,6 @@ class FTS:
For example, it works with `title`, `description`, `content`, etc.
Examples
--------
Create an index configuration that uses 256-document posting blocks:
>>> config = FTS(block_size=256)
Attributes
----------
with_position : bool, default False
@@ -151,18 +145,9 @@ class FTS:
remove_stop_words : bool, default True
Whether to remove stop words. Stop words are common words that are often
removed from text before indexing. For example, in English "the" and "and".
custom_stop_words : list of str, optional
Custom words replace the built-in language stop words
and only take effect when ``remove_stop_words`` is True. ``None`` uses
the built-in language list, while an empty list explicitly uses no
stop words.
ascii_folding : bool, default True
Whether to fold ASCII characters. This converts accented characters to
their ASCII equivalent. For example, "café" would be converted to "cafe".
block_size : int, default 128
The number of documents per compressed posting block. Supported values
are 128 and 256. A value of 256 uses the experimental FTS V3 format
and may introduce breaking changes.
Notes
-----
@@ -183,8 +168,6 @@ class FTS:
ngram_min_length: int = 3
ngram_max_length: int = 3
prefix_only: bool = False
block_size: int = 128
custom_stop_words: Optional[List[str]] = None
@dataclass
@@ -219,7 +202,7 @@ class HnswPq:
distance has a range of (-, ). If the vectors are normalized (i.e. their
l2 norm is 1), then dot distance is equivalent to the cosine distance.
num_partitions: int, default sqrt(num_rows)
num_partitions, default sqrt(num_rows)
The number of IVF partitions to create.
@@ -228,7 +211,7 @@ class HnswPq:
will require too much memory. Each partition becomes its own HNSW graph, so
setting this value higher reduces the peak memory use of training.
num_sub_vectors: int, default is vector dimension / 16
num_sub_vectors, default is vector dimension / 16
Number of sub-vectors of PQ.
@@ -244,13 +227,13 @@ class HnswPq:
If the dimension is not visible by 8 then we use 1 subvector. This is not
ideal and will likely result in poor performance.
num_bits: int, default 8
num_bits: int, default 8
Number of bits to encode each sub-vector.
This value controls how much the sub-vectors are compressed. The more bits
the more accurate the index but the slower search. Only 4 and 8 are supported.
max_iterations: int, default 50
max_iterations, default 50
Max iterations to train kmeans.
@@ -263,7 +246,7 @@ class HnswPq:
those cases it is unlikely that setting this larger will lead to the index
converging anyways.
sample_rate: int, default 256
sample_rate, default 256
The rate used to calculate the number of training vectors for kmeans.
@@ -279,14 +262,14 @@ class HnswPq:
Increasing this value might improve the quality of the index but in
most cases the default should be sufficient.
m: int, default 20
m, default 20
The number of neighbors to select for each vector in the HNSW graph.
This value controls the tradeoff between search speed and accuracy.
The higher the value the more accurate the search but the slower it will be.
ef_construction: int, default 300
ef_construction, default 300
The number of candidates to evaluate during the construction of the HNSW graph.
@@ -297,7 +280,7 @@ class HnswPq:
This value should be set to a value that is not less than `ef` in the
search phase.
target_partition_size: int, default is 1,048,576
target_partition_size, default is 1,048,576
The target size of each partition.
@@ -351,7 +334,7 @@ class HnswSq:
distance has a range of (-, ). If the vectors are normalized (i.e. their
l2 norm is 1), then dot distance is equivalent to the cosine distance.
num_partitions: int, default sqrt(num_rows)
num_partitions, default sqrt(num_rows)
The number of IVF partitions to create.
@@ -360,7 +343,7 @@ class HnswSq:
will require too much memory. Each partition becomes its own HNSW graph, so
setting this value higher reduces the peak memory use of training.
max_iterations: int, default 50
max_iterations, default 50
Max iterations to train kmeans.
@@ -373,7 +356,7 @@ class HnswSq:
In those cases it is unlikely that setting this larger will lead to
the index converging anyways.
sample_rate: int, default 256
sample_rate, default 256
The rate used to calculate the number of training vectors for kmeans.
@@ -389,14 +372,14 @@ class HnswSq:
Increasing this value might improve the quality of the index but in
most cases the default should be sufficient.
m: int, default 20
m, default 20
The number of neighbors to select for each vector in the HNSW graph.
This value controls the tradeoff between search speed and accuracy.
The higher the value the more accurate the search but the slower it will be.
ef_construction: int, default 300
ef_construction, default 300
The number of candidates to evaluate during the construction of the HNSW graph.
@@ -407,7 +390,7 @@ class HnswSq:
This value should be set to a value that is not less than `ef` in the search
phase.
target_partition_size: int, default is 1,048,576
target_partition_size, default is 1,048,576
The target size of each partition.
@@ -460,7 +443,7 @@ class HnswFlat:
distance has a range of (-, ). If the vectors are normalized (i.e. their
l2 norm is 1), then dot distance is equivalent to the cosine distance.
num_partitions: int, default sqrt(num_rows)
num_partitions, default sqrt(num_rows)
The number of IVF partitions to create.
@@ -470,18 +453,18 @@ class HnswFlat:
graph, so setting this value higher reduces the peak memory use of
training.
max_iterations: int, default 50
max_iterations, default 50
Max iterations to train kmeans.
When training an IVF index we use kmeans to calculate the partitions.
This parameter controls how many iterations of kmeans to run.
sample_rate: int, default 256
sample_rate, default 256
The rate used to calculate the number of training vectors for kmeans.
m: int, default 20
m, default 20
The number of neighbors to select for each vector in the HNSW graph.
@@ -489,7 +472,7 @@ class HnswFlat:
The higher the value the more accurate the search but the slower it
will be.
ef_construction: int, default 300
ef_construction, default 300
The number of candidates to evaluate during the construction of the HNSW
graph.
@@ -501,7 +484,7 @@ class HnswFlat:
than 500. This value should be set to a value that is not less than `ef`
in the search phase.
target_partition_size: int, default is 1,048,576
target_partition_size, default is 1,048,576
The target size of each partition.
"""
@@ -605,7 +588,7 @@ class IvfFlat:
The default value is 256.
target_partition_size: int, default is 8192
target_partition_size, default is 8192
The target size of each partition.
@@ -769,7 +752,7 @@ class IvfPq:
The default value is 256.
target_partition_size: int, default is 8192
target_partition_size, default is 8192
The target size of each partition.
@@ -830,7 +813,7 @@ class IvfRq:
sample_rate: int, default 256
Controls the number of training vectors: sample_rate * num_partitions.
target_partition_size: int, default is 8192
target_partition_size, default is 8192
Target size of each partition.
"""
@@ -845,9 +828,6 @@ class IvfRq:
accelerator: Optional[str] = None
# The API reference renders this module with a single mkdocstrings directive,
# which only picks up names listed here. New public names must be added to this
# list, or they will silently go undocumented.
__all__ = [
"BTree",
"IvfPq",
-105
View File
@@ -1,105 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""Handles to operations a server may run asynchronously."""
import asyncio
from datetime import timedelta
from typing import Optional
from lancedb.background_loop import LOOP
from . import _lancedb
class AsyncJob:
"""A handle to an operation that may still be running.
The operation may already be complete when the handle is created.
"""
def __init__(self, inner: Optional["_lancedb.Job"]):
self._inner = inner
@property
def id(self) -> Optional[str]:
"""Identifies the operation on the server that is running it.
Returned for correlating with server logs or the jobs API. Operations
that run in this process have no server id and return `None`. The value
is opaque: parsing it or storing it to resume the job later is not
supported.
"""
return self._inner.id if self._inner is not None else None
async def status(self) -> str:
"""The operation's current lifecycle state: "running", "finished",
"failed", or "cancelled".
A point snapshot; unlike `wait` it does not block or raise on a
terminal failure state. States a newer server reports that this
client version does not know pass through as-is.
"""
if self._inner is None:
return "finished"
return await self._inner.status()
async def wait(self, timeout: Optional[timedelta] = None):
"""Wait until the operation reaches a terminal state.
Raises `JobFailedError` if the operation failed, `JobCancelledError`
if it was cancelled, and `TimeoutError` if `timeout` elapses first.
"""
if self._inner is None:
return
if timeout is None:
await self._inner.wait()
else:
await asyncio.wait_for(self._inner.wait(), timeout.total_seconds())
async def cancel(self):
"""Request cancellation. Cancelling a finished operation is a no-op."""
if self._inner is None:
return
await self._inner.cancel()
class Job:
"""Synchronous counterpart of `AsyncJob`."""
def __init__(self, inner: Optional[AsyncJob]):
self._inner = inner
@property
def id(self) -> Optional[str]:
"""Identifies the operation on the server that is running it.
See :attr:`AsyncJob.id`.
"""
return self._inner.id if self._inner is not None else None
def status(self) -> str:
"""The operation's current lifecycle state: "running", "finished",
"failed", or "cancelled".
See :meth:`AsyncJob.status`.
"""
if self._inner is None:
return "finished"
return LOOP.run(self._inner.status())
def wait(self, timeout: Optional[timedelta] = None):
"""Block until the operation reaches a terminal state.
Raises `JobFailedError` if the operation failed, `JobCancelledError`
if it was cancelled, and `TimeoutError` if `timeout` elapses first.
"""
if self._inner is None:
return
LOOP.run(self._inner.wait(timeout))
def cancel(self):
"""Request cancellation. Cancelling a finished operation is a no-op."""
if self._inner is None:
return
LOOP.run(self._inner.cancel())
+12 -14
View File
@@ -37,7 +37,7 @@ class LanceMergeInsertBuilder(object):
self._when_not_matched_by_source_condition_expr = None
self._timeout = None
self._use_index = True
self._use_lsm = None
self._use_lsm_write = None
self._validate_single_shard = None
def when_matched_update_all(
@@ -92,10 +92,8 @@ class LanceMergeInsertBuilder(object):
self._when_not_matched_by_source_delete = True
if isinstance(condition, Expr):
self._when_not_matched_by_source_condition_expr = condition._inner
self._when_not_matched_by_source_condition = None
else:
elif condition is not None:
self._when_not_matched_by_source_condition = condition
self._when_not_matched_by_source_condition_expr = None
return self
def use_index(self, use_index: bool) -> LanceMergeInsertBuilder:
@@ -115,22 +113,22 @@ class LanceMergeInsertBuilder(object):
self._use_index = use_index
return self
def use_lsm(self, enable: bool) -> LanceMergeInsertBuilder:
def use_lsm_write(self, use_lsm_write: bool) -> LanceMergeInsertBuilder:
"""
Control MemWAL routing for this merge.
Controls whether the merge uses the MemWAL LSM write path.
By default (unset), a `merge_insert` on a table with an LSM write spec is
routed through Lance's MemWAL shard writer, and a table without one uses
the standard path.
By default (unset), a `merge_insert` on a table with an LSM write spec
is routed through Lance's MemWAL shard writer, and a table without one
uses the standard path. Pass `False` to force the standard path even
when a spec is set. Pass `True` to require a spec `merge_insert`
raises an error if none is installed.
Parameters
----------
enable: bool
``True`` forces MemWAL routing and errors if the table has no LSM
write spec. ``False`` forces the standard write path even when a spec
is set.
use_lsm_write: bool
Whether to use the LSM write path.
"""
self._use_lsm = enable
self._use_lsm_write = use_lsm_write
return self
def validate_single_shard(
+1 -95
View File
@@ -38,11 +38,7 @@ from lance_namespace_urllib3_client.models.query_table_request_vector import (
QueryTableRequestVector,
)
from lance_namespace_urllib3_client.models.string_fts_query import StringFtsQuery
from lance_namespace.errors import (
NamespaceNotEmptyError,
NamespaceNotFoundError,
TableNotFoundError,
)
from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError
from lancedb._lancedb import (
connect_namespace as _connect_namespace,
connect_namespace_client as _connect_namespace_client,
@@ -57,8 +53,6 @@ from lance_namespace import (
DropNamespaceResponse,
ListNamespacesResponse,
ListTablesResponse,
NamespaceExistsRequest,
TableExistsRequest,
)
from lancedb.table import AsyncTable, LanceTable, Table
from lancedb.util import validate_table_name
@@ -786,51 +780,6 @@ class LanceNamespaceDBConnection(DBConnection):
"""
return LOOP.run(self._inner.describe_namespace(namespace_path))
@override
def namespace_exists(self, namespace_id: List[str]) -> bool:
"""
Check if a namespace exists.
Parameters
----------
namespace_id : List[str]
The namespace identifier to check.
Returns
-------
bool
True if the namespace exists, False otherwise.
"""
request = NamespaceExistsRequest(id=namespace_id)
try:
self._namespace_client.namespace_exists(request)
return True
except NamespaceNotFoundError:
return False
@override
def table_exists(self, table_id: List[str]) -> bool:
"""
Check if a table exists.
Parameters
----------
table_id : List[str]
The table identifier to check (full path including namespace
segments and table name).
Returns
-------
bool
True if the table exists, False otherwise.
"""
request = TableExistsRequest(id=table_id)
try:
self._namespace_client.table_exists(request)
return True
except TableNotFoundError:
return False
@override
def list_tables(
self,
@@ -1284,49 +1233,6 @@ class AsyncLanceNamespaceDBConnection:
"""
return await self._inner.describe_namespace(namespace_path)
async def namespace_exists(self, namespace_id: List[str]) -> bool:
"""
Check if a namespace exists.
Parameters
----------
namespace_id : List[str]
The namespace identifier to check.
Returns
-------
bool
True if the namespace exists, False otherwise.
"""
request = NamespaceExistsRequest(id=namespace_id)
try:
self._namespace_client.namespace_exists(request)
return True
except NamespaceNotFoundError:
return False
async def table_exists(self, table_id: List[str]) -> bool:
"""
Check if a table exists.
Parameters
----------
table_id : List[str]
The table identifier to check (full path including namespace
segments and table name).
Returns
-------
bool
True if the table exists, False otherwise.
"""
request = TableExistsRequest(id=table_id)
try:
self._namespace_client.table_exists(request)
return True
except TableNotFoundError:
return False
async def list_tables(
self,
namespace_path: Optional[List[str]] = None,
+7 -12
View File
@@ -226,7 +226,7 @@ class PermutationBuilder:
async def do_execute():
inner_tbl = await self._async.execute()
return await LanceTable.from_inner(inner_tbl)
return LanceTable.from_inner(inner_tbl)
return LOOP.run(do_execute())
@@ -438,8 +438,7 @@ class Permutation:
_reader: Optional[PermutationReader] = None,
):
"""
Internal constructor. Use
[from_tables][lancedb.permutation.Permutation.from_tables] instead.
Internal constructor. Use [from_tables](#from_tables) instead.
"""
assert base_table is not None, "base_table is required"
assert selection is not None, "selection is required"
@@ -986,9 +985,8 @@ class Permutation:
types. Conversion of strings, lists, and structs will require creating python
objects and this is not zero-copy.
For custom formatting, use
[with_transform][lancedb.permutation.Permutation.with_transform] which
overrides this method.
For custom formatting, use [with_transform](#with_transform) which overrides
this method.
"""
assert format is not None, "format is required"
if format == "python":
@@ -1063,8 +1061,7 @@ class Permutation:
Note: this method returns a new permutation and does not modify `self`
It is provided for compatibility with the huggingface Dataset API.
Use [with_skip][lancedb.permutation.Permutation.with_skip] instead to
avoid confusion.
Use [with_skip](#with_skip) instead to avoid confusion.
"""
return self.with_skip(skip)
@@ -1087,8 +1084,7 @@ class Permutation:
Note: this method returns a new permutation and does not modify `self`
It is provided for compatibility with the huggingface Dataset API.
Use [with_take][lancedb.permutation.Permutation.with_take] instead to
avoid confusion.
Use [with_take](#with_take) instead to avoid confusion.
"""
return self.with_take(limit)
@@ -1111,8 +1107,7 @@ class Permutation:
Note: this method returns a new permutation and does not modify `self`
It is provided for compatibility with the huggingface Dataset API.
Use [with_repeat][lancedb.permutation.Permutation.with_repeat] instead
to avoid confusion.
Use [with_repeat](#with_repeat) instead to avoid confusion.
"""
return self.with_repeat(times)
+24 -101
View File
@@ -52,6 +52,7 @@ from ._blob import (
finalize_blob_query_table,
replace_v2_blob_columns_with_bytes,
replace_v2_blob_columns_with_bytes_sync,
supports_blob_auto_row_id,
validate_blob_mode,
)
from .types import BlobMode, QueryProjection
@@ -650,8 +651,7 @@ class Query(pydantic.BaseModel):
distance_type : Optional[str]
the distance type to use for vector search
This can be l2 (default), cosine and dot. See
[metric definitions](https://lancedb.com/docs/search/vector-search/) for
This can be l2 (default), cosine and dot. See [metric definitions][search] for
more details.
If this is not a vector search this will be None.
@@ -664,9 +664,8 @@ class Query(pydantic.BaseModel):
- A higher number makes search more accurate but also slower.
- See discussion in
[Querying an ANN Index](https://lancedb.com/docs/indexing/)
for tuning advice.
- See discussion in [Querying an ANN Index][querying-an-ann-index] for
tuning advice.
Will be None if this is not a vector search.
refine_factor : Optional[int]
@@ -674,9 +673,8 @@ class Query(pydantic.BaseModel):
- A higher number makes search more accurate but also slower.
- See discussion in
[Querying an ANN Index](https://lancedb.com/docs/indexing/)
for tuning advice.
- See discussion in [Querying an ANN Index][querying-an-ann-index] for
tuning advice.
Will be None if this is not a vector search.
lower_bound : Optional[float]
@@ -780,11 +778,6 @@ class Query(pydantic.BaseModel):
# if true, will only search the indexed data
fast_search: Optional[bool] = None
# MemWAL LSM read routing: None auto-routes when the table carries a write
# spec, True forces the LSM scanner (errors without a spec), False reads the
# base table only
use_lsm: Optional[bool] = None
# size of the nearest neighbor list maintained during HNSW search
ef: Optional[int] = None
@@ -802,9 +795,6 @@ class Query(pydantic.BaseModel):
query.full_text_query = req.full_text_search
query.columns = req.select
query.with_row_id = req.with_row_id
# use_lsm is a genuine tri-state (None / True / False); preserve it as-is
# so a round-tripped query keeps an explicit False.
query.use_lsm = req.use_lsm
query.vector_column = req.column
query.vector = req.query_vector
query.distance_type = req.distance_type
@@ -977,7 +967,6 @@ class LanceQueryBuilder(ABC):
self._with_row_address = None
self._fragments = None
self._fragment_ids = None
self._use_lsm = None
self._vector = None
self._text = None
self._ef = None
@@ -1279,7 +1268,10 @@ class LanceQueryBuilder(ABC):
return self._with_row_id is True
def _blob_auto_row_id_enabled(self) -> bool:
if not supports_blob_auto_row_id(self._table):
return False
return blob_auto_row_id_for_scan(
self._table,
self._table.schema,
self._columns,
with_row_id=self._with_row_id,
@@ -1334,30 +1326,6 @@ class LanceQueryBuilder(ABC):
self._fragment_ids = fragment_ids
return self
def use_lsm(self, enable: bool) -> Self:
"""Control MemWAL LSM read routing for this query.
By default (unset), a query against a table with an LSM write spec is
routed through the LSM scanner so it also returns data written via the
``merge_insert`` LSM path that has not yet been compacted into the base
table (active/frozen memtables + flushed generations); a table without a
spec reads the base table.
Parameters
----------
enable : bool
``True`` forces the LSM scanner and errors if the table has no LSM
write spec. ``False`` bypasses the MemWAL and reads the base table
only, even when a spec is present.
Returns
-------
LanceQueryBuilder
The LanceQueryBuilder object.
"""
self._use_lsm = enable
return self
def explain_plan(self, verbose: Optional[bool] = False) -> str:
"""Return the execution plan for this query.
@@ -1650,8 +1618,8 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
Higher values will yield better recall (more likely to find vectors if
they exist) at the expense of latency.
See discussion in [Querying an ANN Index](https://lancedb.com/docs/indexing/)
for tuning advice.
See discussion in [Querying an ANN Index][querying-an-ann-index] for
tuning advice.
This method sets both the minimum and maximum number of probes to the same
value. See `minimum_nprobes` and `maximum_nprobes` for more fine-grained
@@ -1751,8 +1719,8 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
As an example, a refine factor of 2 will sample 2x as many vectors as
requested, re-ranks them, and returns the top half most relevant results.
See discussion in [Querying an ANN Index](https://lancedb.com/docs/indexing/)
for tuning advice.
See discussion in [Querying an ANN Index][querying-an-ann-index] for
tuning advice.
Parameters
----------
@@ -1820,7 +1788,6 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
with_row_address=self._with_row_address,
fragments=self._fragments,
fragment_ids=self._fragment_ids,
use_lsm=self._use_lsm,
offset=self._offset,
fast_search=self._fast_search,
ef=self._ef,
@@ -2045,7 +2012,6 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
with_row_address=self._with_row_address,
fragments=self._fragments,
fragment_ids=self._fragment_ids,
use_lsm=self._use_lsm,
full_text_query=FullTextSearchQuery(
query=self._query_with_phrase_semantics(), columns=self._fts_columns
),
@@ -2112,7 +2078,6 @@ class LanceEmptyQueryBuilder(LanceQueryBuilder):
with_row_address=self._with_row_address,
fragments=self._fragments,
fragment_ids=self._fragment_ids,
use_lsm=self._use_lsm,
offset=self._offset,
order_by=self._order_by,
)
@@ -2690,14 +2655,11 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
if self._with_row_id:
self._vector_query.with_row_id(True)
self._fts_query.with_row_id(True)
if self._use_lsm is not None:
self._vector_query.use_lsm(self._use_lsm)
self._fts_query.use_lsm(self._use_lsm)
if self._phrase_query:
self._fts_query.phrase_query(True)
if self._distance_type:
self._vector_query.metric(self._distance_type)
if self._minimum_nprobes is not None:
if self._minimum_nprobes:
self._vector_query.minimum_nprobes(self._minimum_nprobes)
if self._maximum_nprobes is not None:
self._vector_query.maximum_nprobes(self._maximum_nprobes)
@@ -2770,7 +2732,7 @@ class AsyncQueryBase(object):
)
async def _maybe_add_blob_row_id(self) -> None:
if self._table is None:
if self._table is None or not supports_blob_auto_row_id(self._table):
self._blob_auto_row_id = False
self._blob_paths = ()
return
@@ -2778,6 +2740,7 @@ class AsyncQueryBase(object):
req = self._inner.to_query_request()
schema = await self._table.schema()
self._blob_auto_row_id = blob_auto_row_id_for_scan(
self._table,
schema,
req.select,
with_row_id=self._with_row_id,
@@ -3029,6 +2992,7 @@ class AsyncQueryBase(object):
schema = await self._table.schema()
blob_auto_row_id = blob_auto_row_id_for_scan(
self._table,
schema,
query.columns,
with_row_id=self._with_row_id,
@@ -3038,7 +3002,7 @@ class AsyncQueryBase(object):
if blob_mode == "bytes"
else {}
)
dataset = await self._table.to_lance()
dataset = await self._table._to_lance()
scanner = dataset.scanner(
**_scanner_kwargs_for_query(
query,
@@ -3267,27 +3231,6 @@ class AsyncStandardQuery(AsyncQueryBase):
self._inner.fast_search()
return self
def use_lsm(self, enable: bool) -> Self:
"""
Control MemWAL LSM read routing for this query.
By default (unset), a query against a table with an LSM write spec (see
[AsyncTable.set_lsm_write_spec][lancedb.table.AsyncTable.set_lsm_write_spec])
is routed through the LSM scanner so it also returns data written via the
``merge_insert`` LSM path that has not yet been compacted into the base
table (the active/frozen in-memory memtables and the flushed generations),
deduplicated by primary key; a table without a spec reads the base table.
Parameters
----------
enable : bool
``True`` forces the LSM scanner and errors if the table has no LSM
write spec. ``False`` bypasses the MemWAL and reads the base table
only, even when a spec is present.
"""
self._inner.use_lsm(enable)
return self
def postfilter(self) -> Self:
"""
If this is called then filtering will happen after the search instead of
@@ -3376,9 +3319,8 @@ class AsyncQuery(AsyncStandardQuery):
are various ANN search parameters that will let you fine tune your recall
accuracy vs search latency.
Vector searches always have a
[limit][lancedb.query.AsyncVectorQuery.limit]. If `limit` has not been
called then a default `limit` of 10 will be used.
Vector searches always have a [limit][]. If `limit` has not been called then
a default `limit` of 10 will be used.
Typically, a single vector is passed in as the query. However, you can also
pass in multiple vectors. When multiple vectors are passed in, if the vector
@@ -3509,9 +3451,8 @@ class AsyncFTSQuery(AsyncStandardQuery):
are various ANN search parameters that will let you fine tune your recall
accuracy vs search latency.
Hybrid searches always have a
[limit][lancedb.query.AsyncHybridQuery.limit]. If `limit` has not been
called then a default `limit` of 10 will be used.
Hybrid searches always have a [limit][]. If `limit` has not been called then
a default `limit` of 10 will be used.
Typically, a single vector is passed in as the query. However, you can also
pass in multiple vectors. This can be useful if you want to find the nearest
@@ -3874,9 +3815,10 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
req = fts_query._inner.to_query_request()
blob_auto_row_id = False
blob_paths: tuple[str, ...] = ()
if self._table is not None:
if self._table is not None and supports_blob_auto_row_id(self._table):
schema = await self._table.schema()
blob_auto_row_id = blob_auto_row_id_for_scan(
self._table,
schema,
req.select,
with_row_id=self._with_row_id,
@@ -4002,15 +3944,6 @@ class AsyncTakeQuery(AsyncQueryBase):
def __init__(self, inner: LanceTakeQuery, table: Optional["AsyncTable"] = None):
super().__init__(inner, table)
def use_lsm(self, enable: bool) -> "AsyncTakeQuery":
"""Control MemWAL LSM read routing for this take query.
``False`` bypasses the MemWAL and reads the base table only the escape
hatch, since take-by-row-id/offset is not supported on the LSM scanner.
"""
self._inner.use_lsm(enable)
return self
async def _plain_scan_to_pandas(
self,
blob_mode: BlobMode,
@@ -4069,16 +4002,6 @@ class BaseQueryBuilder(object):
self._inner.with_row_id()
return self
def use_lsm(self, enable: bool) -> Self:
"""
Control MemWAL LSM read routing for this query.
``False`` bypasses the MemWAL and reads the base table only, the escape
hatch for shapes the LSM scanner cannot honor (e.g. take-by-row-id).
"""
self._inner.use_lsm(enable)
return self
def with_row_address(self, with_row_address: bool = True) -> Self:
"""
Include the _rowaddr column in scanner-backed plain query results.
-3
View File
@@ -11,9 +11,6 @@ from lancedb import __version__
from .header import HeaderProvider
from .oauth import OAuthConfig, OAuthFlowType
# The API reference renders this module with a single mkdocstrings directive,
# which only picks up names listed here. New public names must be added to this
# list, or they will silently go undocumented.
__all__ = [
"TimeoutConfig",
"RetryConfig",
+1 -51
View File
@@ -7,7 +7,7 @@ import json
import logging
from concurrent.futures import ThreadPoolExecutor
import sys
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
from typing import Any, Dict, Iterable, List, Optional, Union
from urllib.parse import urlparse
import warnings
@@ -23,10 +23,6 @@ import pyarrow as pa
from ..common import DATA
from ..db import DBConnection, LOOP
from ..job import Job
if TYPE_CHECKING:
from .._lancedb import JobDescription, JobInfo
from ..embeddings import EmbeddingFunctionConfig
from lance_namespace import (
LanceNamespace,
@@ -419,11 +415,6 @@ class RemoteDBConnection(DBConnection):
if namespace_path is None:
namespace_path = []
if storage_options is not None:
logging.info(
"storage_options is ignored in LanceDb Cloud"
" (storage is managed; set storage_options on connect() instead)"
)
if index_cache_size is not None:
logging.info(
"index_cache_size is ignored in LanceDb Cloud"
@@ -693,47 +684,6 @@ class RemoteDBConnection(DBConnection):
)
)
@override
def job(self, job_id: str) -> Job:
"""A [Job][lancedb.job.Job] handle for a server-side job by id.
The handle is constructed without a server round trip; an unknown id
surfaces when the handle is used. Dropping the handle has no effect
on the job itself.
"""
return Job(self._conn.job(job_id))
@override
def list_jobs(self) -> List["JobInfo"]:
"""List server-side jobs across the database's tables."""
return LOOP.run(self._conn.list_jobs())
@override
def get_job(self, job_id: str) -> Optional["JobDescription"]:
"""Describe a single server-side job by id.
Returns None when the server has no such job.
"""
return LOOP.run(self._conn.get_job(job_id))
@override
def cancel_job(self, job_id: str) -> bool:
"""Request cancellation of a server-side job by id.
Returns True if the server accepted the cancellation, False if no
such job exists. Cancelling an already-terminal job is a no-op
success.
"""
return LOOP.run(self._conn.cancel_job(job_id))
@override
def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
"""The lifecycle event history of a server-side job, as Arrow batches.
Lists history across all jobs when `job_id` is None.
"""
return LOOP.run(self._conn.job_history(job_id))
@override
def namespace_client(self) -> LanceNamespace:
"""Get the equivalent namespace client for this connection.
+2 -2
View File
@@ -53,9 +53,9 @@ class RetryError(LanceDBClientError):
"""An error that occurs when the client has exceeded the maximum number of retries.
The retry strategy can be adjusted by setting the
[retry_config][lancedb.remote.ClientConfig.retry_config] in the client
[retry_config](lancedb.remote.ClientConfig.retry_config) in the client
configuration. This is passed in the `client_config` argument of
[connect][lancedb.connect] and [connect_async][lancedb.connect_async].
[connect](lancedb.connect) and [connect_async](lancedb.connect_async).
The __cause__ attribute of this exception will be the last exception that
caused the retry to fail. It will be an
+11 -52
View File
@@ -20,7 +20,6 @@ from typing import (
import warnings
from lancedb import __version__
from lancedb._blob import BlobFile
from lancedb._lancedb import (
AddColumnsResult,
@@ -48,7 +47,6 @@ from lancedb.index import (
IvfSq,
LabelList,
)
from lancedb.job import Job
from lancedb.remote.db import LOOP
from lancedb.table import IndexConfigType, KNOWN_METRICS
import pyarrow as pa
@@ -342,12 +340,10 @@ class RemoteTable(Table):
lower_case: bool = True,
stem: bool = True,
remove_stop_words: bool = True,
custom_stop_words: Optional[List[str]] = None,
ascii_folding: bool = True,
ngram_min_length: int = 3,
ngram_max_length: int = 3,
prefix_only: bool = False,
block_size: int = 128,
name: Optional[str] = None,
):
"""Create a full-text search index on a column.
@@ -364,12 +360,10 @@ class RemoteTable(Table):
lower_case=lower_case,
stem=stem,
remove_stop_words=remove_stop_words,
custom_stop_words=custom_stop_words,
ascii_folding=ascii_folding,
ngram_min_length=ngram_min_length,
ngram_max_length=ngram_max_length,
prefix_only=prefix_only,
block_size=block_size,
)
LOOP.run(
self._table.create_index(
@@ -542,34 +536,6 @@ class RemoteTable(Table):
)
)
def create_index_async(
self,
column: str,
*,
config: IndexConfigType,
replace: Optional[bool] = None,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
train: bool = True,
) -> Job:
"""Create an index, returning a handle to the indexing job.
The job may already be complete when returned; callers must not assume
the index exists until :meth:`Job.wait` returns.
"""
return Job(
LOOP.run(
self._table.create_index_async(
column,
replace=replace,
config=config,
wait_timeout=wait_timeout,
name=name,
train=train,
)
)
)
def _is_legacy_create_index_call(
self,
first_arg: str,
@@ -610,9 +576,8 @@ class RemoteTable(Table):
progress: Optional[Union[bool, Callable, Any]] = None,
write_parallelism: Optional[int] = None,
) -> AddResult:
"""Add more data to the [Table][lancedb.table.Table].
It has the same API signature as the OSS version.
"""Add more data to the [Table](Table). It has the same API signature as
the OSS version.
Parameters
----------
@@ -672,8 +637,7 @@ class RemoteTable(Table):
fast_search: bool = False,
) -> LanceVectorQueryBuilder:
"""Create a search query to find the nearest neighbors
of the given query vector. We currently support
[vector search](https://lancedb.com/docs/search/vector-search/)
of the given query vector. We currently support [vector search][search]
All query options are defined in
[LanceVectorQueryBuilder][lancedb.query.LanceVectorQueryBuilder].
@@ -1069,22 +1033,17 @@ class RemoteTable(Table):
)
def blob_columns(self) -> list[str]:
return LOOP.run(self._table.blob_columns())
def fetch_blobs(
self, column: str, row_ids: Union[list[int], pa.Table]
) -> pa.LargeBinaryArray:
return LOOP.run(self._table.fetch_blobs(column, row_ids))
def fetch_blob_ranges(self, column: str, requests) -> pa.LargeBinaryArray:
raise NotImplementedError(
"fetch_blob_ranges() is not supported on LanceDB Cloud"
"blob_columns() is not yet supported on the LanceDB Cloud"
)
def fetch_blob_files(
self, column: str, row_ids: Union[list[int], pa.Table]
) -> "list[Optional[BlobFile]]":
return LOOP.run(self._table.fetch_blob_files(column, row_ids))
def fetch_blobs(self, column: str, row_ids) -> pa.LargeBinaryArray:
raise NotImplementedError("fetch_blobs() is not supported on LanceDB Cloud")
def fetch_blob_files(self, column: str, row_ids):
raise NotImplementedError(
"fetch_blob_files() is not supported on LanceDB Cloud"
)
def head(self, n=5) -> pa.Table:
"""
@@ -14,9 +14,6 @@ from .answerdotai import AnswerdotaiRerankers
from .voyageai import VoyageAIReranker
from .watsonx import WatsonxReranker
# The API reference renders this module with a single mkdocstrings directive,
# which only picks up names listed here. New public names must be added to this
# list, or they will silently go undocumented.
__all__ = [
"Reranker",
"CrossEncoderReranker",
@@ -23,7 +23,7 @@ class AnswerdotaiRerankers(Reranker):
column : str, default "text"
The name of the column to use as input to the cross encoder model.
return_score : str, default "relevance"
options are "relevance" or "all".
options are "relevance" or "all". Only "relevance" is supported for now.
**kwargs
Additional keyword arguments to pass to the model. For example, 'device'.
See AnswerDotAI/rerankers for more information.
@@ -77,13 +77,12 @@ class AnswerdotaiRerankers(Reranker):
vector_results: pa.Table,
fts_results: pa.Table,
):
if self.score == "all":
combined_results = self._merge_and_keep_scores(vector_results, fts_results)
else:
combined_results = self.merge_results(vector_results, fts_results)
combined_results = self.merge_results(vector_results, fts_results)
combined_results = self._rerank(combined_results, query)
if self.score == "relevance":
combined_results = self._keep_relevance_score(combined_results)
elif self.score == "all":
combined_results = self._merge_and_keep_scores(vector_results, fts_results)
combined_results = combined_results.sort_by(
[("_relevance_score", "descending")]
)
+1 -1
View File
@@ -16,7 +16,7 @@ class ColbertReranker(AnswerdotaiRerankers):
column : str, default "text"
The name of the column to use as input to the cross encoder model.
return_score : str, default "relevance"
options are "relevance" or "all".
options are "relevance" or "all". Only "relevance" is supported for now.
**kwargs
Additional keyword arguments to pass to the model, for example, 'device'.
See AnswerDotAI/rerankers for more information.
+8 -21
View File
@@ -59,10 +59,9 @@ class StreamingDataset(IterableDataset):
- **Stage 1 (I/O)**: one thread pool with ``num_splits * prefetch_batches``
workers fetches raw ``RecordBatch`` objects from LanceDB in parallel
across all splits and places them in a per-split raw-batch queue.
- **Stage 2 (transform)**: a second thread pool with
``transform_parallelism`` workers picks up raw batches, applies the
transform, and places the results in a per-split cooked-row queue. By
default, the number of workers is determined by ``os.cpu_count()``.
- **Stage 2 (transform)**: a second thread pool with ``os.cpu_count()``
workers picks up raw batches, applies the transform, and places the
results in a per-split cooked-row queue.
The main thread round-robins over the cooked queues, yielding one row per
split per cycle.
@@ -123,10 +122,6 @@ class StreamingDataset(IterableDataset):
are yielded. Receives one batch at a time and must return an iterable
whose length equals the number of rows in the batch. When ``None``
(the default) rows are returned as plain Python dicts.
transform_parallelism:
Maximum number of transforms to run concurrently. Must be greater
than zero. When ``None`` (the default), uses ``os.cpu_count()`` or 1
when the CPU count is unavailable.
worker_info_override:
If set, used in place of ``torch.utils.data.get_worker_info()`` to
determine the DataLoader worker assignment. Intended for unit tests
@@ -151,7 +146,6 @@ class StreamingDataset(IterableDataset):
shuffle_clump_size: Optional[int] = None,
filter: Optional[str] = None,
transform: Optional[Callable] = None,
transform_parallelism: Optional[int] = None,
connection_factory: Optional[Callable[[str], Any]] = None,
worker_info_override=None,
):
@@ -165,8 +159,6 @@ class StreamingDataset(IterableDataset):
f"num_splits ({num_splits}) must be divisible by "
f"world_size ({world_size})"
)
if transform_parallelism is not None and transform_parallelism <= 0:
raise ValueError("transform_parallelism must be greater than 0")
self._table = table
self._num_splits = num_splits
@@ -181,7 +173,6 @@ class StreamingDataset(IterableDataset):
self._shuffle_clump_size = shuffle_clump_size
self._filter = filter
self._transform = transform
self._transform_parallelism = transform_parallelism
self._connection_factory = connection_factory
self._worker_info_override = worker_info_override
@@ -293,11 +284,7 @@ class StreamingDataset(IterableDataset):
batch_size = self._read_batch_size
max_prefetch = self._prefetch_batches
transform_workers = (
self._transform_parallelism
if self._transform_parallelism is not None
else (os.cpu_count() or 1)
)
cpu_workers = os.cpu_count() or 1
final_transform = (
self._transform if self._transform is not None else Transforms.arrow2python
)
@@ -309,8 +296,8 @@ class StreamingDataset(IterableDataset):
tx_pending = [deque() for _ in range(n)] # Future[list[Any]]
cooked = [deque() for _ in range(n)] # rows ready to yield
# Limit simultaneous transforms to transform_workers across all splits.
tx_semaphore = threading.Semaphore(transform_workers)
# Limit simultaneous transforms to cpu_workers across all splits.
tx_semaphore = threading.Semaphore(cpu_workers)
# ── Stage 1 helpers ───────────────────────────────────────────────────
@@ -382,7 +369,7 @@ class StreamingDataset(IterableDataset):
_advance(i)
elif raw_batches[i]:
# Acquire a transform slot (may block briefly if all
# transform_workers are busy with other splits).
# cpu_workers are busy with other splits).
tx_semaphore.acquire()
batch = raw_batches[i].popleft()
tx_pending[i].append(tx_pool.submit(_tx_call_guarded, batch))
@@ -396,7 +383,7 @@ class StreamingDataset(IterableDataset):
# ── Main loop ─────────────────────────────────────────────────────────
with ThreadPoolExecutor(max_workers=n * max_prefetch) as io_pool:
with ThreadPoolExecutor(max_workers=transform_workers) as tx_pool:
with ThreadPoolExecutor(max_workers=cpu_workers) as tx_pool:
self._raw_batches_ref = raw_batches
self._cooked_ref = cooked
self._fetch_head_ref = fetch_head
+35 -194
View File
@@ -20,7 +20,6 @@ from typing import (
List,
Literal,
Optional,
Sequence,
Tuple,
Union,
overload,
@@ -40,7 +39,6 @@ from ._blob import (
from .types import BlobMode
from lancedb.arrow import peek_reader
from lancedb.background_loop import LOOP, embedding_executor
from lancedb.job import AsyncJob, Job
from .dependencies import (
_check_for_hugging_face,
_check_for_lance,
@@ -978,24 +976,6 @@ class Table(ABC):
"""
raise NotImplementedError
def create_index_async(
self,
column: str,
*,
config: IndexConfigType,
replace: Optional[bool] = None,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
train: bool = True,
) -> Job:
"""Create an index, returning a handle to the indexing job.
Takes the same arguments as :meth:`create_index`. The job may already
be complete when returned; callers must not assume the index exists
until :meth:`Job.wait` returns.
"""
raise NotImplementedError
def drop_index(self, name: str) -> None:
"""
Drop an index from the table.
@@ -1122,12 +1102,10 @@ class Table(ABC):
lower_case: bool = True,
stem: bool = True,
remove_stop_words: bool = True,
custom_stop_words: Optional[List[str]] = None,
ascii_folding: bool = True,
ngram_min_length: int = 3,
ngram_max_length: int = 3,
prefix_only: bool = False,
block_size: int = 128,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
):
@@ -1190,9 +1168,6 @@ class Table(ABC):
remove_stop_words : bool, default True
Whether to remove stop words. Stop words are common words that are often
removed from text before indexing. For example, in English "the" and "and".
custom_stop_words : list of str, optional
Custom words that replace the built-in language stop words. ``None``
uses the built-in list; an empty list explicitly uses no stop words.
ascii_folding : bool, default True
Whether to fold ASCII characters. This converts accented characters to
their ASCII equivalent. For example, "café" would be converted to "cafe".
@@ -1202,10 +1177,6 @@ class Table(ABC):
The maximum length of an n-gram.
prefix_only: bool, default False
Whether to only index the prefix of the token for ngram tokenizer.
block_size: int, default 128
The number of documents per compressed posting block. Must be 128
or 256. A value of 256 uses the experimental FTS V3 format and
may introduce breaking changes.
wait_timeout: timedelta, optional
The timeout to wait if indexing is asynchronous.
name: str, optional
@@ -1230,7 +1201,7 @@ class Table(ABC):
progress: Optional[Union[bool, Callable, Any]] = None,
write_parallelism: Optional[int] = None,
) -> AddResult:
"""Add more data to the [Table][lancedb.table.Table].
"""Add more data to the [Table](Table).
Parameters
----------
@@ -1362,8 +1333,8 @@ class Table(ABC):
fts_columns: Optional[Union[str, List[str]]] = None,
) -> LanceQueryBuilder:
"""Create a search query to find the nearest neighbors
of the given query vector. We currently support [vector search](https://lancedb.com/docs/search/vector-search/)
and [full-text search](https://lancedb.com/docs/search/full-text-search/).
of the given query vector. We currently support [vector search][search]
and [full-text search][experimental-full-text-search].
All query options are defined in
[LanceQueryBuilder][lancedb.query.LanceQueryBuilder].
@@ -1562,30 +1533,10 @@ class Table(ABC):
) -> pa.LargeBinaryArray:
"""Materialize full blob bytes for ``column`` at the given rows.
The result has the same length and order as ``row_ids``. Null blobs
produce null slots; valid empty blobs produce ``b""``.
Convenience for small payloads. For large values use
:meth:`fetch_blob_files`.
"""
@abstractmethod
def fetch_blob_ranges(
self,
column: str,
requests: Sequence[Tuple[int, int, int]],
) -> pa.LargeBinaryArray:
"""Materialize row-specific byte ranges from a blob v2 column.
Each request is a ``(row_id, offset, length)`` tuple. Requests may be
repeated or reordered, including multiple ranges for the same blob.
The result has the same length and order as ``requests``; null blobs
produce null slots and empty ranges on non-null blobs produce ``b""``.
Row IDs can be obtained from a query with ``with_row_id(True)``. This
API is currently supported only by local tables.
"""
@abstractmethod
def fetch_blob_files(
self, column: str, row_ids: Union[list[int], pa.Table]
@@ -1593,10 +1544,8 @@ class Table(ABC):
"""Open lazy, seekable :class:`~lancedb._blob.BlobFile` handles.
Prefer this over :meth:`fetch_blobs` for large payloads. ``row_ids`` is
a ``list[int]`` or a query ``pyarrow.Table`` carrying row identity via
``_rowid`` or a ``_lance_row_id`` field on the blob descriptor. Null
rows are ``None``. Remote tables require LanceDB Cloud server 0.5.0 or
newer.
a ``list[int]`` or query ``pyarrow.Table`` with ``_rowid`` (or stashed
row-id metadata). Null rows are ``None``. Local tables only.
"""
@abstractmethod
@@ -1799,7 +1748,7 @@ class Table(ABC):
for faster reads.
Arguments are passed onto Lance's
`lance.dataset.DatasetOptimizer.compact_files`.
[compact_files][lance.dataset.DatasetOptimizer.compact_files].
For most cases, the default should be fine.
See Also
@@ -1853,8 +1802,6 @@ class Table(ABC):
retrain: bool, default False
This parameter is no longer used and is deprecated.
Notes
-----
The frequency an application should call optimize is based on the frequency of
data modifications. If data is frequently added, deleted, or updated then
optimize should be run frequently. A good rule of thumb is to run optimize if
@@ -2009,14 +1956,15 @@ class Table(ABC):
change permanent you can use the `[Self::restore]` method.
Any operation that modifies the table will fail while the table is in a checked
out state. To return the table to a normal state use
`[Self::checkout_latest]`.
out state.
Parameters
----------
version: int | str,
The version to check out. A version number (`int`) or a tag
(`str`) can be provided.
To return the table to a normal state use `[Self::checkout_latest]`
"""
@abstractmethod
@@ -2182,15 +2130,11 @@ class LanceTable(Table):
return self.name
@classmethod
async def from_inner(cls, tbl: LanceDBTable):
from .db import AsyncConnection, LanceDBConnection
def from_inner(cls, tbl: LanceDBTable):
from .db import LanceDBConnection
async_tbl = AsyncTable(tbl)
inner_conn = tbl.database()
read_consistency_interval = await AsyncConnection(
inner_conn
).get_read_consistency_interval()
conn = LanceDBConnection.from_inner(inner_conn, read_consistency_interval)
conn = LanceDBConnection.from_inner(tbl.database())
return cls(
conn,
async_tbl.name,
@@ -2316,13 +2260,6 @@ class LanceTable(Table):
) -> pa.LargeBinaryArray:
return LOOP.run(self._table.fetch_blobs(column, row_ids))
def fetch_blob_ranges(
self,
column: str,
requests: Sequence[Tuple[int, int, int]],
) -> pa.LargeBinaryArray:
return LOOP.run(self._table.fetch_blob_ranges(column, list(requests)))
def fetch_blob_files(
self, column: str, row_ids: Union[list[int], pa.Table]
) -> "list[Optional[BlobFile]]":
@@ -2494,7 +2431,13 @@ class LanceTable(Table):
return LOOP.run(self._table.count_rows(filter))
def __repr__(self) -> str:
return f"{self.__class__.__name__}(name={self.name!r}, _conn={self._conn!r})"
val = f"{self.__class__.__name__}(name={self.name!r}"
if self._conn.read_consistency_interval is not None:
val += ", read_consistency_interval={!r}".format(
self._conn.read_consistency_interval
)
val += f", _conn={self._conn!r})"
return val
def __str__(self) -> str:
return self.__repr__()
@@ -2803,34 +2746,6 @@ class LanceTable(Table):
)
)
def create_index_async(
self,
column: str,
*,
config: IndexConfigType,
replace: Optional[bool] = None,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
train: bool = True,
) -> Job:
"""Create an index, returning a handle to the indexing job.
The job may already be complete when returned; callers must not assume
the index exists until :meth:`Job.wait` returns.
"""
return Job(
LOOP.run(
self._table.create_index_async(
column,
replace=replace,
config=config,
wait_timeout=wait_timeout,
name=name,
train=train,
)
)
)
def _is_legacy_create_index_call(
self,
first_arg: str,
@@ -3107,12 +3022,10 @@ class LanceTable(Table):
lower_case: bool = True,
stem: bool = True,
remove_stop_words: bool = True,
custom_stop_words: Optional[List[str]] = None,
ascii_folding: bool = True,
ngram_min_length: int = 3,
ngram_max_length: int = 3,
prefix_only: bool = False,
block_size: int = 128,
name: Optional[str] = None,
):
"""Create a full-text search index on a column.
@@ -3154,7 +3067,6 @@ class LanceTable(Table):
"lower_case": lower_case,
"stem": stem,
"remove_stop_words": remove_stop_words,
"custom_stop_words": custom_stop_words,
"ascii_folding": ascii_folding,
"ngram_min_length": ngram_min_length,
"ngram_max_length": ngram_max_length,
@@ -3162,9 +3074,10 @@ class LanceTable(Table):
}
else:
tokenizer_configs = self.infer_tokenizer_configs(tokenizer_name)
tokenizer_configs["custom_stop_words"] = custom_stop_words
config = FTS(block_size=block_size, **tokenizer_configs)
config = FTS(
**tokenizer_configs,
)
try:
LOOP.run(
@@ -3435,8 +3348,8 @@ class LanceTable(Table):
fts_columns: Optional[Union[str, List[str]]] = None,
) -> LanceQueryBuilder:
"""Create a search query to find the nearest neighbors
of the given query vector. We currently support [vector search](https://lancedb.com/docs/search/vector-search/)
and [full-text search](https://lancedb.com/docs/search/full-text-search/).
of the given query vector. We currently support [vector search][search]
and [full-text search][search].
Examples
--------
@@ -3466,9 +3379,8 @@ class LanceTable(Table):
- *default None*.
Acceptable types are: list, np.ndarray, PIL.Image.Image
- If None then the
select/[where][lancedb.query.LanceQueryBuilder.where]/limit clauses
are applied to filter the table
- If None then the select/[where][sql]/limit clauses are applied
to filter the table
vector_column_name: str, optional
The name of the vector column to search.
@@ -3862,8 +3774,6 @@ class LanceTable(Table):
retrain: bool, default False
This parameter is no longer used and is deprecated.
Notes
-----
The frequency an application should call optimize is based on the frequency of
data modifications. If data is frequently added, deleted, or updated then
optimize should be run frequently. A good rule of thumb is to run optimize if
@@ -4736,24 +4646,7 @@ class AsyncTable:
"""
return AsyncQuery(self._inner.query(), self)
async def to_lance(self, **kwargs) -> lance.LanceDataset:
"""Return the Lance dataset backing this table.
Parameters
----------
**kwargs
Forwarded to `lance.dataset`.
Returns
-------
lance.LanceDataset
The Lance dataset at this table handle's version and branch.
Examples
--------
>>> async def get_lance_dataset(table):
... return await table.to_lance()
"""
async def _to_lance(self, **kwargs) -> lance.LanceDataset:
try:
import lance
except ImportError:
@@ -4803,7 +4696,7 @@ class AsyncTable:
return (await self.to_arrow()).to_pandas(**kwargs)
if blob_mode == "bytes" and blob_v2_column_paths(schema):
return await self.query().to_pandas(blob_mode=blob_mode, **kwargs)
return (await self.to_lance()).to_pandas(blob_mode=blob_mode, **kwargs)
return (await self._to_lance()).to_pandas(blob_mode=blob_mode, **kwargs)
async def to_arrow(self) -> pa.Table:
"""Return the table as a pyarrow Table.
@@ -4918,46 +4811,6 @@ class AsyncTable:
)
raise e
async def create_index_async(
self,
column: str,
*,
replace: Optional[bool] = None,
config: Optional[
Union[
IvfFlat,
IvfPq,
IvfRq,
HnswPq,
HnswSq,
HnswFlat,
BTree,
Bitmap,
LabelList,
Fm,
FTS,
]
] = None,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
train: bool = True,
) -> AsyncJob:
"""Create an index, returning a handle to the indexing job.
Takes the same arguments as :meth:`create_index`. The job may already
be complete when returned; callers must not assume the index exists
until :meth:`AsyncJob.wait` resolves.
"""
job = await self._inner.create_index_async(
column,
index=config,
replace=replace,
wait_timeout=wait_timeout,
name=name,
train=train,
)
return AsyncJob(job)
async def drop_index(self, name: str) -> None:
"""
Drop an index from the table.
@@ -5101,7 +4954,7 @@ class AsyncTable:
progress: Optional[Union[bool, Callable, Any]] = None,
write_parallelism: Optional[int] = None,
) -> AddResult:
"""Add more data to the [AsyncTable][lancedb.table.AsyncTable].
"""Add more data to the [Table](Table).
Parameters
----------
@@ -5303,8 +5156,8 @@ class AsyncTable:
fts_columns: Optional[Union[str, List[str]]] = None,
) -> Union[AsyncHybridQuery, AsyncFTSQuery, AsyncVectorQuery]:
"""Create a search query to find the nearest neighbors
of the given query vector. We currently support [vector search](https://lancedb.com/docs/search/vector-search/)
and [full-text search](https://lancedb.com/docs/search/full-text-search/).
of the given query vector. We currently support [vector search][search]
and [full-text search][experimental-full-text-search].
All query options are defined in [AsyncQuery][lancedb.query.AsyncQuery].
@@ -5506,8 +5359,6 @@ class AsyncTable:
async_query = async_query.where(query.filter)
if query.fast_search:
async_query = async_query.fast_search()
if query.use_lsm is not None:
async_query = async_query.use_lsm(query.use_lsm)
if query.with_row_id:
async_query = async_query.with_row_id()
if query.order_by:
@@ -5628,7 +5479,7 @@ class AsyncTable:
when_not_matched_by_source_condition_expr=merge._when_not_matched_by_source_condition_expr,
timeout=merge._timeout,
use_index=merge._use_index,
use_lsm=merge._use_lsm,
use_lsm_write=merge._use_lsm_write,
validate_single_shard=merge._validate_single_shard,
),
)
@@ -5865,14 +5716,15 @@ class AsyncTable:
change permanent you can use the `[Self::restore]` method.
Any operation that modifies the table will fail while the table is in a checked
out state. To return the table to a normal state use
`[Self::checkout_latest]`.
out state.
Parameters
----------
version: int | str,
The version to check out. A version number (`int`) or a tag
(`str`) can be provided.
To return the table to a normal state use `[Self::checkout_latest]`
"""
try:
await self._inner.checkout(version)
@@ -5971,13 +5823,6 @@ class AsyncTable:
column, _normalize_blob_row_ids(row_ids, column)
)
async def fetch_blob_ranges(
self,
column: str,
requests: Sequence[Tuple[int, int, int]],
) -> pa.LargeBinaryArray:
return await self._inner.fetch_blob_ranges(column, list(requests))
async def fetch_blob_files(
self, column: str, row_ids: Union[list[int], pa.Table]
) -> "list[Optional[BlobFile]]":
@@ -6056,8 +5901,6 @@ class AsyncTable:
retrain: bool, default False
This parameter is no longer used and is deprecated.
Notes
-----
The frequency an application should call optimize is based on the frequency of
data modifications. If data is frequently added, deleted, or updated then
optimize should be run frequently. A good rule of thumb is to run optimize if
@@ -6438,8 +6281,6 @@ class Branches:
dry_run: bool, default False
When True, only preview. When False, attempt the merge.
Notes
-----
A rejected merge returns ``status="rejected"`` instead of raising.
"""
return LOOP.run(self._table.branches.merge(from_branch, dry_run))
-4
View File
@@ -102,10 +102,6 @@ def fs_from_uri(uri: str) -> Tuple[pa_fs.FileSystem, str]:
az_blob_fs = adlfs.AzureBlobFileSystem(
account_name=os.environ.get("AZURE_STORAGE_ACCOUNT_NAME"),
account_key=os.environ.get("AZURE_STORAGE_ACCOUNT_KEY"),
# Without an explicit key, authenticate with DefaultAzureCredential
# instead of attempting anonymous access. In particular, this enables
# managed identity authentication on Azure hosts.
anon=False,
)
fs = pa_fs.PyFileSystem(pa_fs.FSSpecHandler(az_blob_fs))
+4 -61
View File
@@ -184,75 +184,18 @@ def test_fetch_blobs_accepts_query_result():
assert {blobs[i].as_py() for i in range(len(blobs))} == {b"gamma"}
def test_fetch_blobs_preserves_null_and_empty_values():
def test_fetch_blobs_null_alignment():
table = _blob_table(
"nulls",
[
{"id": 1, "image": b"present"},
{"id": 2, "image": None},
{"id": 3, "image": b""},
],
[{"id": 1, "image": b"present"}, {"id": 2, "image": None}],
)
by_id = _row_ids_by_id(table)
request = [by_id[1], by_id[2], by_id[3], by_id[1]]
request = [by_id[1], by_id[2], by_id[1]]
blobs = table.fetch_blobs("image", request)
assert len(blobs) == len(request)
assert blobs[0].as_py() == b"present"
assert blobs[1].as_py() is None
assert blobs[2].as_py() == b""
assert blobs[3].as_py() == b"present"
def test_fetch_blob_ranges_aligns_repeated_ranges_and_nulls():
table = _blob_table(
"range_alignment",
[{"id": 1, "image": b"abcdefghij"}, {"id": 2, "image": None}],
)
by_id = _row_ids_by_id(table)
requests = [
(by_id[1], 2, 3),
(by_id[2], 0, 0),
(by_id[1], 0, 2),
(by_id[1], 2, 3),
(by_id[1], 10, 0),
]
ranges = table.fetch_blob_ranges("image", requests)
assert ranges.to_pylist() == [b"cde", None, b"ab", b"cde", b""]
def test_fetch_blob_ranges_validates_requests():
table = _blob_table("range_validation", [{"id": 1, "image": b"abc"}])
row_id = _row_ids_by_id(table)[1]
with pytest.raises(ValueError, match="exceeds blob size"):
table.fetch_blob_ranges("image", [(row_id, 2, 2)])
with pytest.raises(ValueError, match="offset \\+ length overflowed"):
table.fetch_blob_ranges("image", [(row_id, 2**64 - 1, 1)])
with pytest.raises(ValueError, match="row IDs"):
table.fetch_blob_ranges("image", [(2**64 - 1, 0, 1)])
def test_fetch_blob_ranges_empty_requests_returns_empty_array():
table = _blob_table("range_empty", [{"id": 1, "image": b"x"}])
assert table.fetch_blob_ranges("image", []).to_pylist() == []
@pytest.mark.asyncio
async def test_async_fetch_blob_ranges():
db = await lancedb.connect_async("memory:///")
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
table = await db.create_table("range_async", schema=schema)
await table.add([{"id": 1, "image": b"abcdefghij"}])
hits = await table.query().with_row_id().to_arrow()
row_id = hits["_rowid"][0].as_py()
ranges = await table.fetch_blob_ranges("image", [(row_id, 1, 3), (row_id, 6, 2)])
assert ranges.to_pylist() == [b"bcd", b"gh"]
assert blobs[2].as_py() == b"present"
def test_fetch_blobs_nested_path():
-32
View File
@@ -62,38 +62,6 @@ def test_basic(tmp_path):
assert db.open_table("test").name == db["test"].name
def test_sync_repr_does_not_use_background_loop(tmp_path, monkeypatch):
from lancedb.background_loop import LOOP
db = lancedb.connect(tmp_path)
table = db.create_table("test", data=[{"id": 1}])
def fail_run(*args, **kwargs):
raise AssertionError("repr should not use the Python background loop")
monkeypatch.setattr(LOOP, "run", fail_run)
assert repr(db) == f"LanceDBConnection(uri={str(tmp_path)!r})"
assert repr(table) == f"LanceTable(name='test', _conn={db!r})"
def test_read_consistency_interval_does_not_use_background_loop(tmp_path, monkeypatch):
from lancedb.background_loop import LOOP
from lancedb.db import LanceDBConnection
consistency_interval = timedelta(seconds=5)
db = lancedb.connect(tmp_path, read_consistency_interval=consistency_interval)
db_from_inner = LanceDBConnection.from_inner(db._inner, consistency_interval)
def fail_run(*args, **kwargs):
raise AssertionError("properties should not use the Python background loop")
monkeypatch.setattr(LOOP, "run", fail_run)
assert db.read_consistency_interval == consistency_interval
assert db_from_inner.read_consistency_interval == consistency_interval
def test_ingest_pd(tmp_path):
db = lancedb.connect(tmp_path)
@@ -1333,42 +1333,6 @@ def test_transform_none_yields_dicts(lance_table):
assert all("id" in item for item in items)
@pytest.mark.parametrize(
("configured", "detected", "expected"),
[(2, 8, 2), (None, 3, 3), (None, None, 1)],
)
def test_transform_parallelism_configures_executor(
lance_table, monkeypatch, configured, detected, expected
):
"""Explicit transform parallelism overrides the detected CPU count."""
real_executor = streaming.ThreadPoolExecutor
monkeypatch.setattr(streaming.os, "cpu_count", lambda: detected)
with patch.object(streaming, "ThreadPoolExecutor", wraps=real_executor) as executor:
list(
StreamingDataset(
lance_table,
num_splits=NUM_SPLITS,
shuffle_seed=SHUFFLE_SEED,
transform_parallelism=configured,
)
)
assert executor.call_args_list[-1].kwargs["max_workers"] == expected
@pytest.mark.parametrize("transform_parallelism", [0, -1])
def test_transform_parallelism_must_be_positive(lance_table, transform_parallelism):
with pytest.raises(
ValueError, match="transform_parallelism must be greater than 0"
):
StreamingDataset(
lance_table,
num_splits=NUM_SPLITS,
transform_parallelism=transform_parallelism,
)
def test_filter_limits_rows(tmp_path):
"""A filter expression is applied to the permutation so only matching rows
are yielded. IDs 0..59 pass ``id < 60``; the other 60 are excluded."""
-37
View File
@@ -219,48 +219,11 @@ def test_create_inverted_index(table, with_position):
table.create_fts_index(
"text",
with_position=with_position,
custom_stop_words=["puppy"],
name="custom_fts_index",
)
indices = table.list_indices()
fts_indices = [i for i in indices if i.index_type == "FTS"]
assert any(i.name == "custom_fts_index" for i in fts_indices)
assert fts_indices[0].index_details["custom_stop_words"] == ["puppy"]
@pytest.mark.parametrize("block_size", [128, 256])
def test_create_inverted_index_block_size(table, block_size):
table.create_index("text", config=FTS(block_size=block_size))
index = next(index for index in table.list_indices() if index.index_type == "FTS")
assert index.index_details["block_size"] == block_size
assert index.index_version == (2 if block_size == 128 else 3)
results = table.search("puppy").limit(5).to_list()
assert len(results) == 5
def test_create_inverted_index_rejects_invalid_block_size(table):
with pytest.raises(ValueError, match="128 or 256"):
table.create_index("text", config=FTS(block_size=129))
def test_custom_stop_words_list(table):
table.create_index(
"text",
config=FTS(stem=False, custom_stop_words=["lance"]),
)
assert table.list_indices()[0].index_details["custom_stop_words"] == ["lance"]
tokens = table.tokenize("the lance data", column="text")
assert [token.text for token in tokens] == ["the", "data"]
empty_tokens = ldb.tokenize("the lance data", stem=False, custom_stop_words=[])
assert [token.text for token in empty_tokens] == ["the", "lance", "data"]
with pytest.raises(TypeError, match=r"custom_stop_words.*int"):
ldb.tokenize(
"the lance data",
custom_stop_words=["lance", 42],
)
def test_search_fts(table):

Some files were not shown because too many files have changed in this diff Show More