mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-30 01:48:19 +00:00
Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Jack Ye
|
f54f5600ad | ||
|
Jack Ye
|
e34fe84c7f | ||
|
Jack Ye
|
5b1f248257 | ||
|
Jack Ye
|
95e34d47b9 | ||
|
Jack Ye
|
a0defd448f | ||
|
Jack Ye
|
0fadb65153 | ||
|
Jack Ye
|
15fbcf61fc |
+1
-1
@@ -1,5 +1,5 @@
|
|||||||
[tool.bumpversion]
|
[tool.bumpversion]
|
||||||
current_version = "0.28.0-beta.9"
|
current_version = "0.28.0-beta.7"
|
||||||
parse = """(?x)
|
parse = """(?x)
|
||||||
(?P<major>0|[1-9]\\d*)\\.
|
(?P<major>0|[1-9]\\d*)\\.
|
||||||
(?P<minor>0|[1-9]\\d*)\\.
|
(?P<minor>0|[1-9]\\d*)\\.
|
||||||
|
|||||||
@@ -1,18 +0,0 @@
|
|||||||
version: 2
|
|
||||||
|
|
||||||
# Scope: the root Cargo workspace, which produces the Rust binaries we
|
|
||||||
# ship to users (the Node.js and Python native extensions). The
|
|
||||||
# `rust/lancedb` library crate shares the same lockfile; its consumers
|
|
||||||
# pick their own dependency versions, but bumping transitive deps here
|
|
||||||
# keeps the binaries we ship current.
|
|
||||||
updates:
|
|
||||||
- package-ecosystem: cargo
|
|
||||||
directory: /
|
|
||||||
schedule:
|
|
||||||
interval: weekly
|
|
||||||
open-pull-requests-limit: 10
|
|
||||||
groups:
|
|
||||||
rust-minor-patch:
|
|
||||||
update-types:
|
|
||||||
- minor
|
|
||||||
- patch
|
|
||||||
@@ -8,9 +8,6 @@ concurrency:
|
|||||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||||
cancel-in-progress: true
|
cancel-in-progress: true
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
labeler:
|
labeler:
|
||||||
permissions:
|
permissions:
|
||||||
|
|||||||
@@ -19,9 +19,6 @@ on:
|
|||||||
paths:
|
paths:
|
||||||
- .github/workflows/java-publish.yml
|
- .github/workflows/java-publish.yml
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
publish:
|
publish:
|
||||||
name: Build and Publish
|
name: Build and Publish
|
||||||
|
|||||||
@@ -24,9 +24,6 @@ on:
|
|||||||
- java/**
|
- java/**
|
||||||
- .github/workflows/java.yml
|
- .github/workflows/java.yml
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
build-java:
|
build-java:
|
||||||
runs-on: ubuntu-24.04
|
runs-on: ubuntu-24.04
|
||||||
|
|||||||
@@ -10,10 +10,6 @@ on:
|
|||||||
- nodejs/**
|
- nodejs/**
|
||||||
- java/**
|
- java/**
|
||||||
- .github/workflows/license-header-check.yml
|
- .github/workflows/license-header-check.yml
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
check-licenses:
|
check-licenses:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
|||||||
@@ -15,9 +15,6 @@ on:
|
|||||||
- .github/workflows/nodejs.yml
|
- .github/workflows/nodejs.yml
|
||||||
- docker-compose.yml
|
- docker-compose.yml
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
|
|
||||||
concurrency:
|
concurrency:
|
||||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||||
cancel-in-progress: true
|
cancel-in-progress: true
|
||||||
|
|||||||
@@ -14,16 +14,10 @@ on:
|
|||||||
env:
|
env:
|
||||||
PIP_EXTRA_INDEX_URL: "https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/"
|
PIP_EXTRA_INDEX_URL: "https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/"
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
linux:
|
linux:
|
||||||
name: Python ${{ matrix.config.platform }} manylinux${{ matrix.config.manylinux }}
|
name: Python ${{ matrix.config.platform }} manylinux${{ matrix.config.manylinux }}
|
||||||
timeout-minutes: 60
|
timeout-minutes: 60
|
||||||
permissions:
|
|
||||||
id-token: write
|
|
||||||
contents: read
|
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
config:
|
config:
|
||||||
@@ -63,12 +57,10 @@ jobs:
|
|||||||
- uses: ./.github/workflows/upload_wheel
|
- uses: ./.github/workflows/upload_wheel
|
||||||
if: startsWith(github.ref, 'refs/tags/python-v')
|
if: startsWith(github.ref, 'refs/tags/python-v')
|
||||||
with:
|
with:
|
||||||
|
pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
|
||||||
fury_token: ${{ secrets.FURY_TOKEN }}
|
fury_token: ${{ secrets.FURY_TOKEN }}
|
||||||
mac:
|
mac:
|
||||||
timeout-minutes: 90
|
timeout-minutes: 90
|
||||||
permissions:
|
|
||||||
id-token: write
|
|
||||||
contents: read
|
|
||||||
runs-on: ${{ matrix.config.runner }}
|
runs-on: ${{ matrix.config.runner }}
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
@@ -93,12 +85,10 @@ jobs:
|
|||||||
- uses: ./.github/workflows/upload_wheel
|
- uses: ./.github/workflows/upload_wheel
|
||||||
if: startsWith(github.ref, 'refs/tags/python-v')
|
if: startsWith(github.ref, 'refs/tags/python-v')
|
||||||
with:
|
with:
|
||||||
|
pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
|
||||||
fury_token: ${{ secrets.FURY_TOKEN }}
|
fury_token: ${{ secrets.FURY_TOKEN }}
|
||||||
windows:
|
windows:
|
||||||
timeout-minutes: 60
|
timeout-minutes: 60
|
||||||
permissions:
|
|
||||||
id-token: write
|
|
||||||
contents: read
|
|
||||||
runs-on: windows-latest
|
runs-on: windows-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
@@ -117,6 +107,7 @@ jobs:
|
|||||||
- uses: ./.github/workflows/upload_wheel
|
- uses: ./.github/workflows/upload_wheel
|
||||||
if: startsWith(github.ref, 'refs/tags/python-v')
|
if: startsWith(github.ref, 'refs/tags/python-v')
|
||||||
with:
|
with:
|
||||||
|
pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
|
||||||
fury_token: ${{ secrets.FURY_TOKEN }}
|
fury_token: ${{ secrets.FURY_TOKEN }}
|
||||||
gh-release:
|
gh-release:
|
||||||
if: startsWith(github.ref, 'refs/tags/python-v')
|
if: startsWith(github.ref, 'refs/tags/python-v')
|
||||||
|
|||||||
@@ -17,9 +17,6 @@ on:
|
|||||||
- .github/workflows/build_windows_wheel/**
|
- .github/workflows/build_windows_wheel/**
|
||||||
- .github/workflows/run_tests/**
|
- .github/workflows/run_tests/**
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
|
|
||||||
concurrency:
|
concurrency:
|
||||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||||
cancel-in-progress: true
|
cancel-in-progress: true
|
||||||
@@ -111,6 +108,7 @@ jobs:
|
|||||||
- name: Install
|
- name: Install
|
||||||
run: |
|
run: |
|
||||||
pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests,dev,embeddings]
|
pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests,dev,embeddings]
|
||||||
|
pip install tantivy
|
||||||
pip install mlx
|
pip install mlx
|
||||||
- name: Doctest
|
- name: Doctest
|
||||||
run: pytest --doctest-modules python/lancedb
|
run: pytest --doctest-modules python/lancedb
|
||||||
@@ -229,5 +227,6 @@ jobs:
|
|||||||
pip install "pydantic<2"
|
pip install "pydantic<2"
|
||||||
pip install pyarrow==16
|
pip install pyarrow==16
|
||||||
pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests]
|
pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests]
|
||||||
|
pip install tantivy
|
||||||
- name: Run tests
|
- name: Run tests
|
||||||
run: pytest -m "not slow and not s3_test" -x -v --durations=30 python/tests
|
run: pytest -m "not slow and not s3_test" -x -v --durations=30 python/tests
|
||||||
|
|||||||
@@ -9,15 +9,9 @@ on:
|
|||||||
- Cargo.toml
|
- Cargo.toml
|
||||||
- Cargo.lock
|
- Cargo.lock
|
||||||
- rust-toolchain.toml
|
- rust-toolchain.toml
|
||||||
- deny.toml
|
|
||||||
- rust/**
|
- rust/**
|
||||||
- nodejs/Cargo.toml
|
|
||||||
- python/Cargo.toml
|
|
||||||
- .github/workflows/rust.yml
|
- .github/workflows/rust.yml
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
|
|
||||||
concurrency:
|
concurrency:
|
||||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||||
cancel-in-progress: true
|
cancel-in-progress: true
|
||||||
@@ -59,17 +53,6 @@ jobs:
|
|||||||
- name: Run clippy (without remote feature)
|
- name: Run clippy (without remote feature)
|
||||||
run: cargo clippy --profile ci --workspace --tests -- -D warnings
|
run: cargo clippy --profile ci --workspace --tests -- -D warnings
|
||||||
|
|
||||||
deny:
|
|
||||||
# Supply-chain checks: advisories, licenses, banned crates, and source
|
|
||||||
# restrictions. Configuration lives in `deny.toml` at the workspace root.
|
|
||||||
timeout-minutes: 10
|
|
||||||
runs-on: ubuntu-24.04
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
- uses: EmbarkStudios/cargo-deny-action@v2
|
|
||||||
with:
|
|
||||||
command: check advisories bans licenses sources
|
|
||||||
|
|
||||||
build-no-lock:
|
build-no-lock:
|
||||||
runs-on: ubuntu-24.04
|
runs-on: ubuntu-24.04
|
||||||
timeout-minutes: 30
|
timeout-minutes: 30
|
||||||
|
|||||||
@@ -3,9 +3,6 @@ name: Update package-lock.json
|
|||||||
on:
|
on:
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
publish:
|
publish:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
|||||||
@@ -3,9 +3,6 @@ name: Update NodeJs package-lock.json
|
|||||||
on:
|
on:
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
publish:
|
publish:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
|||||||
@@ -2,6 +2,9 @@ name: upload-wheel
|
|||||||
|
|
||||||
description: "Upload wheels to Pypi"
|
description: "Upload wheels to Pypi"
|
||||||
inputs:
|
inputs:
|
||||||
|
pypi_token:
|
||||||
|
required: true
|
||||||
|
description: "release token for the repo"
|
||||||
fury_token:
|
fury_token:
|
||||||
required: true
|
required: true
|
||||||
description: "release token for the fury repo"
|
description: "release token for the fury repo"
|
||||||
@@ -9,6 +12,12 @@ inputs:
|
|||||||
runs:
|
runs:
|
||||||
using: "composite"
|
using: "composite"
|
||||||
steps:
|
steps:
|
||||||
|
- name: Install dependencies
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
python -m pip install --upgrade pip
|
||||||
|
pip install twine
|
||||||
|
python3 -m pip install --upgrade pkginfo
|
||||||
- name: Choose repo
|
- name: Choose repo
|
||||||
shell: bash
|
shell: bash
|
||||||
id: choose_repo
|
id: choose_repo
|
||||||
@@ -18,17 +27,19 @@ runs:
|
|||||||
else
|
else
|
||||||
echo "repo=pypi" >> $GITHUB_OUTPUT
|
echo "repo=pypi" >> $GITHUB_OUTPUT
|
||||||
fi
|
fi
|
||||||
- name: Publish to Fury
|
- name: Publish to PyPI
|
||||||
if: steps.choose_repo.outputs.repo == 'fury'
|
|
||||||
shell: bash
|
shell: bash
|
||||||
env:
|
env:
|
||||||
FURY_TOKEN: ${{ inputs.fury_token }}
|
FURY_TOKEN: ${{ inputs.fury_token }}
|
||||||
|
PYPI_TOKEN: ${{ inputs.pypi_token }}
|
||||||
run: |
|
run: |
|
||||||
WHEEL=$(ls target/wheels/lancedb-*.whl 2> /dev/null | head -n 1)
|
if [[ ${{ steps.choose_repo.outputs.repo }} == fury ]]; then
|
||||||
echo "Uploading $WHEEL to Fury"
|
WHEEL=$(ls target/wheels/lancedb-*.whl 2> /dev/null | head -n 1)
|
||||||
curl -f -F package=@$WHEEL https://$FURY_TOKEN@push.fury.io/lancedb/
|
echo "Uploading $WHEEL to Fury"
|
||||||
- name: Publish to PyPI
|
curl -f -F package=@$WHEEL https://$FURY_TOKEN@push.fury.io/lancedb/
|
||||||
if: steps.choose_repo.outputs.repo == 'pypi'
|
else
|
||||||
uses: pypa/gh-action-pypi-publish@release/v1
|
twine upload --repository ${{ steps.choose_repo.outputs.repo }} \
|
||||||
with:
|
--username __token__ \
|
||||||
packages-dir: target/wheels/
|
--password $PYPI_TOKEN \
|
||||||
|
target/wheels/lancedb-*.whl
|
||||||
|
fi
|
||||||
|
|||||||
Generated
+91
-91
@@ -572,9 +572,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "aws-lc-rs"
|
name = "aws-lc-rs"
|
||||||
version = "1.16.3"
|
version = "1.16.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f"
|
checksum = "94bffc006df10ac2a68c83692d734a465f8ee6c5b384d8545a636f81d858f4bf"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aws-lc-sys",
|
"aws-lc-sys",
|
||||||
"untrusted 0.7.1",
|
"untrusted 0.7.1",
|
||||||
@@ -583,9 +583,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "aws-lc-sys"
|
name = "aws-lc-sys"
|
||||||
version = "0.40.0"
|
version = "0.38.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7"
|
checksum = "4321e568ed89bb5a7d291a7f37997c2c0df89809d7b6d12062c81ddb54aa782e"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"cc",
|
"cc",
|
||||||
"cmake",
|
"cmake",
|
||||||
@@ -1373,7 +1373,7 @@ dependencies = [
|
|||||||
"memmap2 0.9.10",
|
"memmap2 0.9.10",
|
||||||
"num-traits",
|
"num-traits",
|
||||||
"num_cpus",
|
"num_cpus",
|
||||||
"rand 0.9.4",
|
"rand 0.9.2",
|
||||||
"rand_distr 0.5.1",
|
"rand_distr 0.5.1",
|
||||||
"rayon",
|
"rayon",
|
||||||
"safetensors",
|
"safetensors",
|
||||||
@@ -1409,7 +1409,7 @@ dependencies = [
|
|||||||
"candle-nn",
|
"candle-nn",
|
||||||
"fancy-regex",
|
"fancy-regex",
|
||||||
"num-traits",
|
"num-traits",
|
||||||
"rand 0.9.4",
|
"rand 0.9.2",
|
||||||
"rayon",
|
"rayon",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
@@ -1966,7 +1966,7 @@ dependencies = [
|
|||||||
"log",
|
"log",
|
||||||
"object_store",
|
"object_store",
|
||||||
"parking_lot",
|
"parking_lot",
|
||||||
"rand 0.9.4",
|
"rand 0.9.2",
|
||||||
"regex",
|
"regex",
|
||||||
"sqlparser 0.59.0",
|
"sqlparser 0.59.0",
|
||||||
"tempfile",
|
"tempfile",
|
||||||
@@ -2080,7 +2080,7 @@ dependencies = [
|
|||||||
"itertools 0.14.0",
|
"itertools 0.14.0",
|
||||||
"log",
|
"log",
|
||||||
"object_store",
|
"object_store",
|
||||||
"rand 0.9.4",
|
"rand 0.9.2",
|
||||||
"tokio",
|
"tokio",
|
||||||
"url",
|
"url",
|
||||||
]
|
]
|
||||||
@@ -2176,7 +2176,7 @@ dependencies = [
|
|||||||
"log",
|
"log",
|
||||||
"object_store",
|
"object_store",
|
||||||
"parking_lot",
|
"parking_lot",
|
||||||
"rand 0.9.4",
|
"rand 0.9.2",
|
||||||
"tempfile",
|
"tempfile",
|
||||||
"url",
|
"url",
|
||||||
]
|
]
|
||||||
@@ -2240,7 +2240,7 @@ dependencies = [
|
|||||||
"log",
|
"log",
|
||||||
"md-5",
|
"md-5",
|
||||||
"num-traits",
|
"num-traits",
|
||||||
"rand 0.9.4",
|
"rand 0.9.2",
|
||||||
"regex",
|
"regex",
|
||||||
"sha2",
|
"sha2",
|
||||||
"unicode-segmentation",
|
"unicode-segmentation",
|
||||||
@@ -2642,7 +2642,7 @@ dependencies = [
|
|||||||
"libc",
|
"libc",
|
||||||
"option-ext",
|
"option-ext",
|
||||||
"redox_users",
|
"redox_users",
|
||||||
"windows-sys 0.61.2",
|
"windows-sys 0.59.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -2830,7 +2830,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
"windows-sys 0.61.2",
|
"windows-sys 0.59.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -2965,7 +2965,7 @@ checksum = "719a903cc23e4a89e87962c2a80fdb45cdaad0983a89bd150bb57b4c8571a7d5"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"half",
|
"half",
|
||||||
"num-traits",
|
"num-traits",
|
||||||
"rand 0.9.4",
|
"rand 0.9.2",
|
||||||
"rand_distr 0.5.1",
|
"rand_distr 0.5.1",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -3010,11 +3010,11 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "fsst"
|
name = "fsst"
|
||||||
version = "6.0.0-beta.4"
|
version = "6.0.0-beta.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.4#226dafe0a00f995d9e3230c2d61cc06be51994d4"
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
"rand 0.9.4",
|
"rand 0.9.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -3387,7 +3387,7 @@ dependencies = [
|
|||||||
"cfg-if",
|
"cfg-if",
|
||||||
"crunchy",
|
"crunchy",
|
||||||
"num-traits",
|
"num-traits",
|
||||||
"rand 0.9.4",
|
"rand 0.9.2",
|
||||||
"rand_distr 0.5.1",
|
"rand_distr 0.5.1",
|
||||||
"zerocopy",
|
"zerocopy",
|
||||||
]
|
]
|
||||||
@@ -3470,7 +3470,7 @@ dependencies = [
|
|||||||
"libc",
|
"libc",
|
||||||
"log",
|
"log",
|
||||||
"num_cpus",
|
"num_cpus",
|
||||||
"rand 0.9.4",
|
"rand 0.9.2",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
@@ -3980,7 +3980,7 @@ dependencies = [
|
|||||||
"portable-atomic",
|
"portable-atomic",
|
||||||
"portable-atomic-util",
|
"portable-atomic-util",
|
||||||
"serde_core",
|
"serde_core",
|
||||||
"windows-sys 0.61.2",
|
"windows-sys 0.59.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -4043,7 +4043,7 @@ dependencies = [
|
|||||||
"nom 8.0.0",
|
"nom 8.0.0",
|
||||||
"num-traits",
|
"num-traits",
|
||||||
"ordered-float",
|
"ordered-float",
|
||||||
"rand 0.9.4",
|
"rand 0.9.2",
|
||||||
"ryu",
|
"ryu",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
@@ -4066,8 +4066,8 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance"
|
name = "lance"
|
||||||
version = "6.0.0-beta.4"
|
version = "6.0.0-beta.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.4#226dafe0a00f995d9e3230c2d61cc06be51994d4"
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"arrow-arith",
|
"arrow-arith",
|
||||||
@@ -4119,7 +4119,7 @@ dependencies = [
|
|||||||
"prost",
|
"prost",
|
||||||
"prost-build",
|
"prost-build",
|
||||||
"prost-types",
|
"prost-types",
|
||||||
"rand 0.9.4",
|
"rand 0.9.2",
|
||||||
"roaring",
|
"roaring",
|
||||||
"semver",
|
"semver",
|
||||||
"serde",
|
"serde",
|
||||||
@@ -4135,8 +4135,8 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-arrow"
|
name = "lance-arrow"
|
||||||
version = "6.0.0-beta.4"
|
version = "6.0.0-beta.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.4#226dafe0a00f995d9e3230c2d61cc06be51994d4"
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
"arrow-buffer",
|
"arrow-buffer",
|
||||||
@@ -4152,13 +4152,13 @@ dependencies = [
|
|||||||
"half",
|
"half",
|
||||||
"jsonb",
|
"jsonb",
|
||||||
"num-traits",
|
"num-traits",
|
||||||
"rand 0.9.4",
|
"rand 0.9.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-bitpacking"
|
name = "lance-bitpacking"
|
||||||
version = "6.0.0-beta.4"
|
version = "6.0.0-beta.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.4#226dafe0a00f995d9e3230c2d61cc06be51994d4"
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrayref",
|
"arrayref",
|
||||||
"paste",
|
"paste",
|
||||||
@@ -4167,8 +4167,8 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-core"
|
name = "lance-core"
|
||||||
version = "6.0.0-beta.4"
|
version = "6.0.0-beta.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.4#226dafe0a00f995d9e3230c2d61cc06be51994d4"
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
"arrow-buffer",
|
"arrow-buffer",
|
||||||
@@ -4191,7 +4191,7 @@ dependencies = [
|
|||||||
"object_store",
|
"object_store",
|
||||||
"pin-project",
|
"pin-project",
|
||||||
"prost",
|
"prost",
|
||||||
"rand 0.9.4",
|
"rand 0.9.2",
|
||||||
"roaring",
|
"roaring",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"snafu 0.9.0",
|
"snafu 0.9.0",
|
||||||
@@ -4205,8 +4205,8 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-datafusion"
|
name = "lance-datafusion"
|
||||||
version = "6.0.0-beta.4"
|
version = "6.0.0-beta.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.4#226dafe0a00f995d9e3230c2d61cc06be51994d4"
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
@@ -4237,8 +4237,8 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-datagen"
|
name = "lance-datagen"
|
||||||
version = "6.0.0-beta.4"
|
version = "6.0.0-beta.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.4#226dafe0a00f995d9e3230c2d61cc06be51994d4"
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
@@ -4248,7 +4248,7 @@ dependencies = [
|
|||||||
"futures",
|
"futures",
|
||||||
"half",
|
"half",
|
||||||
"hex",
|
"hex",
|
||||||
"rand 0.9.4",
|
"rand 0.9.2",
|
||||||
"rand_distr 0.5.1",
|
"rand_distr 0.5.1",
|
||||||
"rand_xoshiro",
|
"rand_xoshiro",
|
||||||
"random_word 0.5.2",
|
"random_word 0.5.2",
|
||||||
@@ -4256,8 +4256,8 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-encoding"
|
name = "lance-encoding"
|
||||||
version = "6.0.0-beta.4"
|
version = "6.0.0-beta.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.4#226dafe0a00f995d9e3230c2d61cc06be51994d4"
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow-arith",
|
"arrow-arith",
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
@@ -4283,7 +4283,7 @@ dependencies = [
|
|||||||
"prost",
|
"prost",
|
||||||
"prost-build",
|
"prost-build",
|
||||||
"prost-types",
|
"prost-types",
|
||||||
"rand 0.9.4",
|
"rand 0.9.2",
|
||||||
"snafu 0.9.0",
|
"snafu 0.9.0",
|
||||||
"strum",
|
"strum",
|
||||||
"tokio",
|
"tokio",
|
||||||
@@ -4294,8 +4294,8 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-file"
|
name = "lance-file"
|
||||||
version = "6.0.0-beta.4"
|
version = "6.0.0-beta.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.4#226dafe0a00f995d9e3230c2d61cc06be51994d4"
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow-arith",
|
"arrow-arith",
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
@@ -4327,8 +4327,8 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-index"
|
name = "lance-index"
|
||||||
version = "6.0.0-beta.4"
|
version = "6.0.0-beta.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.4#226dafe0a00f995d9e3230c2d61cc06be51994d4"
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"arrow-arith",
|
"arrow-arith",
|
||||||
@@ -4374,7 +4374,7 @@ dependencies = [
|
|||||||
"prost",
|
"prost",
|
||||||
"prost-build",
|
"prost-build",
|
||||||
"prost-types",
|
"prost-types",
|
||||||
"rand 0.9.4",
|
"rand 0.9.2",
|
||||||
"rand_distr 0.5.1",
|
"rand_distr 0.5.1",
|
||||||
"rangemap",
|
"rangemap",
|
||||||
"rayon",
|
"rayon",
|
||||||
@@ -4392,8 +4392,8 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-io"
|
name = "lance-io"
|
||||||
version = "6.0.0-beta.4"
|
version = "6.0.0-beta.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.4#226dafe0a00f995d9e3230c2d61cc06be51994d4"
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"arrow-arith",
|
"arrow-arith",
|
||||||
@@ -4426,7 +4426,7 @@ dependencies = [
|
|||||||
"path_abs",
|
"path_abs",
|
||||||
"pin-project",
|
"pin-project",
|
||||||
"prost",
|
"prost",
|
||||||
"rand 0.9.4",
|
"rand 0.9.2",
|
||||||
"serde",
|
"serde",
|
||||||
"snafu 0.9.0",
|
"snafu 0.9.0",
|
||||||
"tempfile",
|
"tempfile",
|
||||||
@@ -4437,8 +4437,8 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-linalg"
|
name = "lance-linalg"
|
||||||
version = "6.0.0-beta.4"
|
version = "6.0.0-beta.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.4#226dafe0a00f995d9e3230c2d61cc06be51994d4"
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
"arrow-buffer",
|
"arrow-buffer",
|
||||||
@@ -4449,13 +4449,13 @@ dependencies = [
|
|||||||
"lance-arrow",
|
"lance-arrow",
|
||||||
"lance-core",
|
"lance-core",
|
||||||
"num-traits",
|
"num-traits",
|
||||||
"rand 0.9.4",
|
"rand 0.9.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-namespace"
|
name = "lance-namespace"
|
||||||
version = "6.0.0-beta.4"
|
version = "6.0.0-beta.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.4#226dafe0a00f995d9e3230c2d61cc06be51994d4"
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
@@ -4468,8 +4468,8 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-namespace-impls"
|
name = "lance-namespace-impls"
|
||||||
version = "6.0.0-beta.4"
|
version = "6.0.0-beta.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.4#226dafe0a00f995d9e3230c2d61cc06be51994d4"
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"arrow-ipc",
|
"arrow-ipc",
|
||||||
@@ -4488,7 +4488,7 @@ dependencies = [
|
|||||||
"lance-table",
|
"lance-table",
|
||||||
"log",
|
"log",
|
||||||
"object_store",
|
"object_store",
|
||||||
"rand 0.9.4",
|
"rand 0.9.2",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
@@ -4501,9 +4501,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-namespace-reqwest-client"
|
name = "lance-namespace-reqwest-client"
|
||||||
version = "0.7.2"
|
version = "0.6.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "0f061dd6fe63e3ba4052702a9d40973ee4ac57f612f04222897a149576213832"
|
checksum = "ee2e48de899e2931afb67fcddd0a08e439bf5d8b6ea2a2ed9cb8f4df669bd5cc"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"reqwest",
|
"reqwest",
|
||||||
"serde",
|
"serde",
|
||||||
@@ -4514,8 +4514,8 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-table"
|
name = "lance-table"
|
||||||
version = "6.0.0-beta.4"
|
version = "6.0.0-beta.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.4#226dafe0a00f995d9e3230c2d61cc06be51994d4"
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
@@ -4539,7 +4539,7 @@ dependencies = [
|
|||||||
"prost",
|
"prost",
|
||||||
"prost-build",
|
"prost-build",
|
||||||
"prost-types",
|
"prost-types",
|
||||||
"rand 0.9.4",
|
"rand 0.9.2",
|
||||||
"rangemap",
|
"rangemap",
|
||||||
"roaring",
|
"roaring",
|
||||||
"semver",
|
"semver",
|
||||||
@@ -4554,20 +4554,20 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-testing"
|
name = "lance-testing"
|
||||||
version = "6.0.0-beta.4"
|
version = "6.0.0-beta.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.4#226dafe0a00f995d9e3230c2d61cc06be51994d4"
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
"arrow-schema",
|
"arrow-schema",
|
||||||
"lance-arrow",
|
"lance-arrow",
|
||||||
"num-traits",
|
"num-traits",
|
||||||
"rand 0.9.4",
|
"rand 0.9.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-tokenizer"
|
name = "lance-tokenizer"
|
||||||
version = "6.0.0-beta.4"
|
version = "6.0.0-beta.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.4#226dafe0a00f995d9e3230c2d61cc06be51994d4"
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"rust-stemmers",
|
"rust-stemmers",
|
||||||
"serde",
|
"serde",
|
||||||
@@ -4576,7 +4576,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lancedb"
|
name = "lancedb"
|
||||||
version = "0.28.0-beta.9"
|
version = "0.28.0-beta.7"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"ahash",
|
"ahash",
|
||||||
"anyhow",
|
"anyhow",
|
||||||
@@ -4637,7 +4637,7 @@ dependencies = [
|
|||||||
"pin-project",
|
"pin-project",
|
||||||
"polars",
|
"polars",
|
||||||
"polars-arrow",
|
"polars-arrow",
|
||||||
"rand 0.9.4",
|
"rand 0.9.2",
|
||||||
"random_word 0.4.3",
|
"random_word 0.4.3",
|
||||||
"regex",
|
"regex",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
@@ -4658,7 +4658,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lancedb-nodejs"
|
name = "lancedb-nodejs"
|
||||||
version = "0.28.0-beta.9"
|
version = "0.28.0-beta.7"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
"arrow-buffer",
|
"arrow-buffer",
|
||||||
@@ -4680,7 +4680,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lancedb-python"
|
name = "lancedb-python"
|
||||||
version = "0.31.0-beta.9"
|
version = "0.31.0-beta.7"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
@@ -5235,7 +5235,7 @@ version = "0.50.3"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
|
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"windows-sys 0.61.2",
|
"windows-sys 0.59.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -5357,7 +5357,7 @@ dependencies = [
|
|||||||
"parking_lot",
|
"parking_lot",
|
||||||
"percent-encoding",
|
"percent-encoding",
|
||||||
"quick-xml 0.38.4",
|
"quick-xml 0.38.4",
|
||||||
"rand 0.9.4",
|
"rand 0.9.2",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
"ring",
|
"ring",
|
||||||
"rustls-pemfile",
|
"rustls-pemfile",
|
||||||
@@ -6199,8 +6199,8 @@ version = "0.14.3"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7"
|
checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"heck 0.4.1",
|
"heck 0.5.0",
|
||||||
"itertools 0.14.0",
|
"itertools 0.11.0",
|
||||||
"log",
|
"log",
|
||||||
"multimap",
|
"multimap",
|
||||||
"petgraph",
|
"petgraph",
|
||||||
@@ -6219,7 +6219,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b"
|
checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"itertools 0.14.0",
|
"itertools 0.11.0",
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
"syn 2.0.117",
|
"syn 2.0.117",
|
||||||
@@ -6402,7 +6402,7 @@ dependencies = [
|
|||||||
"bytes",
|
"bytes",
|
||||||
"getrandom 0.3.4",
|
"getrandom 0.3.4",
|
||||||
"lru-slab",
|
"lru-slab",
|
||||||
"rand 0.9.4",
|
"rand 0.9.2",
|
||||||
"ring",
|
"ring",
|
||||||
"rustc-hash",
|
"rustc-hash",
|
||||||
"rustls 0.23.37",
|
"rustls 0.23.37",
|
||||||
@@ -6425,7 +6425,7 @@ dependencies = [
|
|||||||
"once_cell",
|
"once_cell",
|
||||||
"socket2 0.6.3",
|
"socket2 0.6.3",
|
||||||
"tracing",
|
"tracing",
|
||||||
"windows-sys 0.60.2",
|
"windows-sys 0.59.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -6468,9 +6468,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rand"
|
name = "rand"
|
||||||
version = "0.9.4"
|
version = "0.9.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea"
|
checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"rand_chacha 0.9.0",
|
"rand_chacha 0.9.0",
|
||||||
"rand_core 0.9.5",
|
"rand_core 0.9.5",
|
||||||
@@ -6531,7 +6531,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463"
|
checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"num-traits",
|
"num-traits",
|
||||||
"rand 0.9.4",
|
"rand 0.9.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -6566,7 +6566,7 @@ dependencies = [
|
|||||||
"ahash",
|
"ahash",
|
||||||
"brotli 8.0.2",
|
"brotli 8.0.2",
|
||||||
"paste",
|
"paste",
|
||||||
"rand 0.9.4",
|
"rand 0.9.2",
|
||||||
"unicase",
|
"unicase",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -6954,7 +6954,7 @@ dependencies = [
|
|||||||
"errno",
|
"errno",
|
||||||
"libc",
|
"libc",
|
||||||
"linux-raw-sys",
|
"linux-raw-sys",
|
||||||
"windows-sys 0.61.2",
|
"windows-sys 0.59.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -6980,7 +6980,7 @@ dependencies = [
|
|||||||
"once_cell",
|
"once_cell",
|
||||||
"ring",
|
"ring",
|
||||||
"rustls-pki-types",
|
"rustls-pki-types",
|
||||||
"rustls-webpki 0.103.13",
|
"rustls-webpki 0.103.10",
|
||||||
"subtle",
|
"subtle",
|
||||||
"zeroize",
|
"zeroize",
|
||||||
]
|
]
|
||||||
@@ -7028,9 +7028,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rustls-webpki"
|
name = "rustls-webpki"
|
||||||
version = "0.103.13"
|
version = "0.103.10"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
|
checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aws-lc-rs",
|
"aws-lc-rs",
|
||||||
"ring",
|
"ring",
|
||||||
@@ -7465,7 +7465,7 @@ version = "0.8.9"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451"
|
checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"heck 0.4.1",
|
"heck 0.5.0",
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
"syn 2.0.117",
|
"syn 2.0.117",
|
||||||
@@ -7477,7 +7477,7 @@ version = "0.9.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40"
|
checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"heck 0.4.1",
|
"heck 0.5.0",
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
"syn 2.0.117",
|
"syn 2.0.117",
|
||||||
@@ -7818,7 +7818,7 @@ dependencies = [
|
|||||||
"getrandom 0.4.2",
|
"getrandom 0.4.2",
|
||||||
"once_cell",
|
"once_cell",
|
||||||
"rustix",
|
"rustix",
|
||||||
"windows-sys 0.61.2",
|
"windows-sys 0.59.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -8242,7 +8242,7 @@ version = "2.1.2"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c"
|
checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"rand 0.9.4",
|
"rand 0.9.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -8298,9 +8298,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "unicode-segmentation"
|
name = "unicode-segmentation"
|
||||||
version = "1.13.2"
|
version = "1.13.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c"
|
checksum = "da36089a805484bcccfffe0739803392c8298778a2d2f09febf76fac5ad9025b"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "unicode-width"
|
name = "unicode-width"
|
||||||
@@ -8632,7 +8632,7 @@ version = "0.1.11"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"windows-sys 0.61.2",
|
"windows-sys 0.59.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|||||||
+16
-14
@@ -1,5 +1,7 @@
|
|||||||
[workspace]
|
[workspace]
|
||||||
members = ["rust/lancedb", "nodejs", "python"]
|
members = ["rust/lancedb", "nodejs", "python"]
|
||||||
|
# Python package needs to be built by maturin.
|
||||||
|
exclude = ["python"]
|
||||||
resolver = "2"
|
resolver = "2"
|
||||||
|
|
||||||
[workspace.package]
|
[workspace.package]
|
||||||
@@ -13,20 +15,20 @@ categories = ["database-implementations"]
|
|||||||
rust-version = "1.91.0"
|
rust-version = "1.91.0"
|
||||||
|
|
||||||
[workspace.dependencies]
|
[workspace.dependencies]
|
||||||
lance = { "version" = "=6.0.0-beta.4", default-features = false, "tag" = "v6.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
lance = { "version" = "=6.0.0-beta.1", default-features = false, "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||||
lance-core = { "version" = "=6.0.0-beta.4", "tag" = "v6.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
lance-core = { "version" = "=6.0.0-beta.1", "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||||
lance-datagen = { "version" = "=6.0.0-beta.4", "tag" = "v6.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
lance-datagen = { "version" = "=6.0.0-beta.1", "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||||
lance-file = { "version" = "=6.0.0-beta.4", "tag" = "v6.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
lance-file = { "version" = "=6.0.0-beta.1", "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||||
lance-io = { "version" = "=6.0.0-beta.4", default-features = false, "tag" = "v6.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
lance-io = { "version" = "=6.0.0-beta.1", default-features = false, "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||||
lance-index = { "version" = "=6.0.0-beta.4", "tag" = "v6.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
lance-index = { "version" = "=6.0.0-beta.1", "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||||
lance-linalg = { "version" = "=6.0.0-beta.4", "tag" = "v6.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
lance-linalg = { "version" = "=6.0.0-beta.1", "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||||
lance-namespace = { "version" = "=6.0.0-beta.4", "tag" = "v6.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
lance-namespace = { "version" = "=6.0.0-beta.1", "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||||
lance-namespace-impls = { "version" = "=6.0.0-beta.4", default-features = false, "tag" = "v6.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
lance-namespace-impls = { "version" = "=6.0.0-beta.1", default-features = false, "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||||
lance-table = { "version" = "=6.0.0-beta.4", "tag" = "v6.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
lance-table = { "version" = "=6.0.0-beta.1", "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||||
lance-testing = { "version" = "=6.0.0-beta.4", "tag" = "v6.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
lance-testing = { "version" = "=6.0.0-beta.1", "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||||
lance-datafusion = { "version" = "=6.0.0-beta.4", "tag" = "v6.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
lance-datafusion = { "version" = "=6.0.0-beta.1", "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||||
lance-encoding = { "version" = "=6.0.0-beta.4", "tag" = "v6.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
lance-encoding = { "version" = "=6.0.0-beta.1", "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||||
lance-arrow = { "version" = "=6.0.0-beta.4", "tag" = "v6.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
lance-arrow = { "version" = "=6.0.0-beta.1", "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||||
ahash = "0.8"
|
ahash = "0.8"
|
||||||
# Note that this one does not include pyarrow
|
# Note that this one does not include pyarrow
|
||||||
arrow = { version = "57.2", optional = false }
|
arrow = { version = "57.2", optional = false }
|
||||||
|
|||||||
@@ -1,172 +0,0 @@
|
|||||||
# cargo-deny configuration for LanceDB.
|
|
||||||
#
|
|
||||||
# Run locally with `cargo deny check`. See
|
|
||||||
# https://embarkstudios.github.io/cargo-deny/ for the full reference.
|
|
||||||
|
|
||||||
# The set of target triples we care about. cargo-deny will only consider
|
|
||||||
# dependencies that are used on at least one of these targets. Keeping this
|
|
||||||
# explicit avoids noise from platform-specific crates (e.g. wasm, android,
|
|
||||||
# ios) that we never actually ship.
|
|
||||||
[graph]
|
|
||||||
targets = [
|
|
||||||
"x86_64-unknown-linux-gnu",
|
|
||||||
"aarch64-unknown-linux-gnu",
|
|
||||||
"x86_64-apple-darwin",
|
|
||||||
"aarch64-apple-darwin",
|
|
||||||
"x86_64-pc-windows-msvc",
|
|
||||||
"aarch64-pc-windows-msvc",
|
|
||||||
]
|
|
||||||
all-features = true
|
|
||||||
|
|
||||||
[output]
|
|
||||||
feature-depth = 1
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Advisories: security vulnerabilities and yanked crates.
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
[advisories]
|
|
||||||
version = 2
|
|
||||||
# Fail the check if any crate in the lockfile has been yanked from crates.io.
|
|
||||||
# Yanked crates are a signal the author retracted the release (often due to
|
|
||||||
# bugs or security issues) and should not be depended on.
|
|
||||||
yanked = "deny"
|
|
||||||
# Advisory IDs we have explicitly reviewed and chosen to accept. Every
|
|
||||||
# entry must include a rationale and, where possible, an upstream issue
|
|
||||||
# pointing to a fix. Revisit this list whenever dependencies are updated.
|
|
||||||
ignore = [
|
|
||||||
# rsa: Marvin Attack timing side-channel in PKCS#1 v1.5 decryption.
|
|
||||||
# Reached only through opendal → reqsign → rsa. We do not use RSA
|
|
||||||
# decryption in LanceDB ourselves; this is dormant in the signing path.
|
|
||||||
# No fixed release exists upstream as of this writing.
|
|
||||||
# https://rustsec.org/advisories/RUSTSEC-2023-0071
|
|
||||||
{ id = "RUSTSEC-2023-0071", reason = "rsa crate via opendal/reqsign; no fixed upstream release" },
|
|
||||||
|
|
||||||
# instant: unmaintained. Pulled in via backoff → instant. Upstream
|
|
||||||
# recommends switching to `web-time`; fix has to come from backoff.
|
|
||||||
# https://rustsec.org/advisories/RUSTSEC-2024-0384
|
|
||||||
{ id = "RUSTSEC-2024-0384", reason = "transitive via backoff; waiting on backoff replacement" },
|
|
||||||
|
|
||||||
# paste: unmaintained (author archived the repo). Used transitively by
|
|
||||||
# datafusion and the arrow ecosystem; widespread, no drop-in replacement.
|
|
||||||
# https://rustsec.org/advisories/RUSTSEC-2024-0436
|
|
||||||
{ id = "RUSTSEC-2024-0436", reason = "transitive via datafusion; awaiting ecosystem migration" },
|
|
||||||
|
|
||||||
# tantivy: segfault on malformed input due to missing bounds check.
|
|
||||||
# Pulled in via lance for full-text search. We only feed tantivy
|
|
||||||
# documents we construct ourselves, not attacker-controlled bytes.
|
|
||||||
# Tracked for a lance dependency bump.
|
|
||||||
# https://rustsec.org/advisories/RUSTSEC-2025-0003
|
|
||||||
{ id = "RUSTSEC-2025-0003", reason = "tantivy via lance; inputs are internally produced, not user-supplied bytes" },
|
|
||||||
|
|
||||||
# backoff: unmaintained. Reached only via async-openai. Replacement
|
|
||||||
# requires async-openai to migrate (or us to drop async-openai).
|
|
||||||
# https://rustsec.org/advisories/RUSTSEC-2025-0012
|
|
||||||
{ id = "RUSTSEC-2025-0012", reason = "transitive via async-openai; waiting on upstream migration" },
|
|
||||||
|
|
||||||
# number_prefix: unmaintained. Transitive via indicatif → hf-hub.
|
|
||||||
# No security impact, just maintenance status.
|
|
||||||
# https://rustsec.org/advisories/RUSTSEC-2025-0119
|
|
||||||
{ id = "RUSTSEC-2025-0119", reason = "transitive via hf-hub/indicatif; cosmetic formatting crate" },
|
|
||||||
|
|
||||||
# rustls-pemfile: unmaintained. Reached from two separate chains:
|
|
||||||
# rustls-native-certs 0.6 (via hyper-rustls 0.24) and object_store 0.12.
|
|
||||||
# Both upstream dependencies need to move before we can drop it.
|
|
||||||
# https://rustsec.org/advisories/RUSTSEC-2025-0134
|
|
||||||
{ id = "RUSTSEC-2025-0134", reason = "transitive via rustls-native-certs/object_store; waiting on upstream migration" },
|
|
||||||
|
|
||||||
# rustls-webpki 0.101.7 (old major line): name-constraint checks for
|
|
||||||
# URI / wildcard names. Pulled in only via the legacy rustls 0.21 chain
|
|
||||||
# from aws-smithy-http-client. The 0.103 line we actively use is patched.
|
|
||||||
# Clearing the 0.101 copy requires the aws-sdk chain to migrate off
|
|
||||||
# rustls 0.21.
|
|
||||||
# https://rustsec.org/advisories/RUSTSEC-2026-0098
|
|
||||||
# https://rustsec.org/advisories/RUSTSEC-2026-0099
|
|
||||||
{ id = "RUSTSEC-2026-0098", reason = "only affects rustls-webpki 0.101 from legacy aws-smithy/rustls 0.21 chain" },
|
|
||||||
{ id = "RUSTSEC-2026-0099", reason = "only affects rustls-webpki 0.101 from legacy aws-smithy/rustls 0.21 chain" },
|
|
||||||
|
|
||||||
# rustls-webpki 0.101.7: reachable panic in CRL parsing. Same legacy
|
|
||||||
# rustls 0.21 chain from aws-smithy-http-client as above. The 0.103 line
|
|
||||||
# we actively use is upgraded to 0.103.13 which contains the fix.
|
|
||||||
# https://rustsec.org/advisories/RUSTSEC-2026-0104
|
|
||||||
{ id = "RUSTSEC-2026-0104", reason = "only affects rustls-webpki 0.101 from legacy aws-smithy/rustls 0.21 chain" },
|
|
||||||
]
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Licenses: only allow licenses we've reviewed as compatible with Apache-2.0.
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
[licenses]
|
|
||||||
version = 2
|
|
||||||
# SPDX identifiers for licenses that are compatible with our Apache-2.0
|
|
||||||
# distribution. Additions require legal review.
|
|
||||||
allow = [
|
|
||||||
"Apache-2.0",
|
|
||||||
"Apache-2.0 WITH LLVM-exception",
|
|
||||||
"MIT",
|
|
||||||
"BSD-2-Clause",
|
|
||||||
"BSD-3-Clause",
|
|
||||||
"ISC",
|
|
||||||
"Unicode-3.0",
|
|
||||||
"Unicode-DFS-2016",
|
|
||||||
"Zlib",
|
|
||||||
"CC0-1.0",
|
|
||||||
"MPL-2.0",
|
|
||||||
"BSL-1.0",
|
|
||||||
"OpenSSL",
|
|
||||||
# 0BSD ("BSD Zero Clause") is effectively public domain — no attribution
|
|
||||||
# required. Pulled in by `mock_instant`.
|
|
||||||
"0BSD",
|
|
||||||
# bzip2-1.0.6 is the permissive upstream bzip2 license (BSD-like). Pulled
|
|
||||||
# in by `libbz2-rs-sys`, the pure-Rust bzip2 implementation.
|
|
||||||
"bzip2-1.0.6",
|
|
||||||
# CDLA-Permissive-2.0 is a permissive data license used by `webpki-roots`
|
|
||||||
# for the Mozilla CA root bundle. Data-only, distribution-compatible.
|
|
||||||
"CDLA-Permissive-2.0",
|
|
||||||
]
|
|
||||||
confidence-threshold = 0.8
|
|
||||||
# Crates whose license cannot be determined from Cargo metadata but whose
|
|
||||||
# license we've manually confirmed from upstream. Keep this list minimal.
|
|
||||||
[[licenses.clarify]]
|
|
||||||
# polars-arrow-format omits the `license` field in its Cargo.toml, but the
|
|
||||||
# upstream repo (pola-rs/polars-arrow-format) is dual-licensed Apache-2.0 OR
|
|
||||||
# MIT. See https://github.com/pola-rs/polars-arrow-format/blob/main/LICENSE
|
|
||||||
crate = "polars-arrow-format"
|
|
||||||
expression = "Apache-2.0 OR MIT"
|
|
||||||
license-files = []
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Bans: disallow specific crates and flag dependency hygiene issues.
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
[bans]
|
|
||||||
# Warn (not deny) on duplicate versions of the same crate. In a large
|
|
||||||
# workspace like this one, duplicates are common and often unavoidable
|
|
||||||
# transitively. We surface them to discourage growth, but don't fail CI.
|
|
||||||
multiple-versions = "warn"
|
|
||||||
# Wildcard version requirements (`foo = "*"`) are a footgun — they let any
|
|
||||||
# future release in without review. Ban them outright.
|
|
||||||
wildcards = "deny"
|
|
||||||
# Internal workspace crates reference each other via `path = "..."`, which
|
|
||||||
# cargo-deny sees as a wildcard version. That's fine for private workspace
|
|
||||||
# members (not published to crates.io), so allow it specifically for paths.
|
|
||||||
allow-wildcard-paths = true
|
|
||||||
# Features that, if enabled, should cause the check to fail.
|
|
||||||
deny = []
|
|
||||||
# Crates to skip when checking for duplicate versions.
|
|
||||||
skip = []
|
|
||||||
# Similar to `skip`, but also skips the entire transitive subtree.
|
|
||||||
skip-tree = []
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Sources: restrict where crates can come from.
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
[sources]
|
|
||||||
# Deny any registry other than the ones explicitly listed below.
|
|
||||||
unknown-registry = "deny"
|
|
||||||
# Deny any git dependency whose host isn't in the allow-list below. This
|
|
||||||
# prevents accidental pulls from arbitrary forks.
|
|
||||||
unknown-git = "deny"
|
|
||||||
allow-registry = ["https://github.com/rust-lang/crates.io-index"]
|
|
||||||
# Lance is developed in a sibling repo and pulled as a git dependency until
|
|
||||||
# releases are cut to crates.io. Allow that specific host.
|
|
||||||
allow-git = [
|
|
||||||
"https://github.com/lance-format/lance",
|
|
||||||
]
|
|
||||||
@@ -24,4 +24,4 @@ RUN python --version && \
|
|||||||
rustc --version && \
|
rustc --version && \
|
||||||
protoc --version
|
protoc --version
|
||||||
|
|
||||||
RUN pip install --no-cache-dir lancedb
|
RUN pip install --no-cache-dir tantivy lancedb
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
|
|||||||
<dependency>
|
<dependency>
|
||||||
<groupId>com.lancedb</groupId>
|
<groupId>com.lancedb</groupId>
|
||||||
<artifactId>lancedb-core</artifactId>
|
<artifactId>lancedb-core</artifactId>
|
||||||
<version>0.28.0-beta.9</version>
|
<version>0.28.0-beta.7</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
@@ -8,7 +8,7 @@
|
|||||||
<parent>
|
<parent>
|
||||||
<groupId>com.lancedb</groupId>
|
<groupId>com.lancedb</groupId>
|
||||||
<artifactId>lancedb-parent</artifactId>
|
<artifactId>lancedb-parent</artifactId>
|
||||||
<version>0.28.0-beta.9</version>
|
<version>0.28.0-beta.7</version>
|
||||||
<relativePath>../pom.xml</relativePath>
|
<relativePath>../pom.xml</relativePath>
|
||||||
</parent>
|
</parent>
|
||||||
|
|
||||||
|
|||||||
+2
-2
@@ -6,7 +6,7 @@
|
|||||||
|
|
||||||
<groupId>com.lancedb</groupId>
|
<groupId>com.lancedb</groupId>
|
||||||
<artifactId>lancedb-parent</artifactId>
|
<artifactId>lancedb-parent</artifactId>
|
||||||
<version>0.28.0-beta.9</version>
|
<version>0.28.0-beta.7</version>
|
||||||
<packaging>pom</packaging>
|
<packaging>pom</packaging>
|
||||||
<name>${project.artifactId}</name>
|
<name>${project.artifactId}</name>
|
||||||
<description>LanceDB Java SDK Parent POM</description>
|
<description>LanceDB Java SDK Parent POM</description>
|
||||||
@@ -28,7 +28,7 @@
|
|||||||
<properties>
|
<properties>
|
||||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||||
<arrow.version>15.0.0</arrow.version>
|
<arrow.version>15.0.0</arrow.version>
|
||||||
<lance-core.version>6.0.0-beta.4</lance-core.version>
|
<lance-core.version>6.0.0-beta.1</lance-core.version>
|
||||||
<spotless.skip>false</spotless.skip>
|
<spotless.skip>false</spotless.skip>
|
||||||
<spotless.version>2.30.0</spotless.version>
|
<spotless.version>2.30.0</spotless.version>
|
||||||
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
|
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
|
||||||
|
|||||||
+3
-4
@@ -1,8 +1,7 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "lancedb-nodejs"
|
name = "lancedb-nodejs"
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
version = "0.28.0-beta.9"
|
version = "0.28.0-beta.7"
|
||||||
publish = false
|
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
description.workspace = true
|
description.workspace = true
|
||||||
repository.workspace = true
|
repository.workspace = true
|
||||||
@@ -32,8 +31,8 @@ lzma-sys = { version = "0.1", features = ["static"] }
|
|||||||
log.workspace = true
|
log.workspace = true
|
||||||
|
|
||||||
# Pin to resolve build failures; update periodically for security patches.
|
# Pin to resolve build failures; update periodically for security patches.
|
||||||
aws-lc-sys = "=0.40.0"
|
aws-lc-sys = "=0.38.0"
|
||||||
aws-lc-rs = "=1.16.3"
|
aws-lc-rs = "=1.16.1"
|
||||||
|
|
||||||
[build-dependencies]
|
[build-dependencies]
|
||||||
napi-build = "2.3.1"
|
napi-build = "2.3.1"
|
||||||
|
|||||||
@@ -1,8 +1,6 @@
|
|||||||
// SPDX-License-Identifier: Apache-2.0
|
// SPDX-License-Identifier: Apache-2.0
|
||||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||||
|
|
||||||
import { spawn } from "node:child_process";
|
|
||||||
import * as path from "node:path";
|
|
||||||
import { RecordBatch } from "apache-arrow";
|
import { RecordBatch } from "apache-arrow";
|
||||||
import * as tmp from "tmp";
|
import * as tmp from "tmp";
|
||||||
import { Connection, Index, Table, connect, makeArrowTable } from "../lancedb";
|
import { Connection, Index, Table, connect, makeArrowTable } from "../lancedb";
|
||||||
@@ -78,91 +76,4 @@ describe("rerankers", function () {
|
|||||||
|
|
||||||
expect(result).toHaveLength(2);
|
expect(result).toHaveLength(2);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("does not keep process alive after rerank query", async function () {
|
|
||||||
const script = `
|
|
||||||
import * as lancedb from "./dist/index.js";
|
|
||||||
import * as os from "node:os";
|
|
||||||
import * as path from "node:path";
|
|
||||||
import * as fs from "node:fs/promises";
|
|
||||||
|
|
||||||
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "lancedb-rerank-exit-"));
|
|
||||||
const db = await lancedb.connect(dir);
|
|
||||||
const table = await db.createTable("test", [{ text: "hello", vector: [1, 2, 3] }], {
|
|
||||||
mode: "overwrite",
|
|
||||||
});
|
|
||||||
await table.createIndex("text", { config: lancedb.Index.fts() });
|
|
||||||
await table.waitForIndex(["text_idx"], 30);
|
|
||||||
|
|
||||||
const reranker = await lancedb.rerankers.RRFReranker.create();
|
|
||||||
await table
|
|
||||||
.query()
|
|
||||||
.nearestTo([1, 2, 3])
|
|
||||||
.fullTextSearch("hello")
|
|
||||||
.rerank(reranker)
|
|
||||||
.toArray();
|
|
||||||
|
|
||||||
table.close();
|
|
||||||
db.close();
|
|
||||||
`;
|
|
||||||
|
|
||||||
await new Promise<void>((resolve, reject) => {
|
|
||||||
const child = spawn(
|
|
||||||
process.execPath,
|
|
||||||
["--input-type=module", "-e", script],
|
|
||||||
{
|
|
||||||
cwd: path.resolve(__dirname, ".."),
|
|
||||||
stdio: ["ignore", "pipe", "pipe"],
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
let stdout = "";
|
|
||||||
let stderr = "";
|
|
||||||
|
|
||||||
child.stdout.on("data", (chunk) => {
|
|
||||||
stdout += chunk.toString();
|
|
||||||
});
|
|
||||||
|
|
||||||
child.stderr.on("data", (chunk) => {
|
|
||||||
stderr += chunk.toString();
|
|
||||||
});
|
|
||||||
|
|
||||||
const timeout = setTimeout(() => {
|
|
||||||
child.kill();
|
|
||||||
reject(
|
|
||||||
new Error(
|
|
||||||
`child process did not exit in time\nstdout:\n${stdout}\nstderr:\n${stderr}`,
|
|
||||||
),
|
|
||||||
);
|
|
||||||
}, 20_000);
|
|
||||||
|
|
||||||
child.on("error", (err) => {
|
|
||||||
clearTimeout(timeout);
|
|
||||||
reject(err);
|
|
||||||
});
|
|
||||||
|
|
||||||
child.on("exit", (code, signal) => {
|
|
||||||
clearTimeout(timeout);
|
|
||||||
if (signal !== null) {
|
|
||||||
reject(
|
|
||||||
new Error(
|
|
||||||
`child process exited with signal ${signal}\nstdout:\n${stdout}\nstderr:\n${stderr}`,
|
|
||||||
),
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (code !== 0) {
|
|
||||||
reject(
|
|
||||||
new Error(
|
|
||||||
`child process exited with code ${code}\nstdout:\n${stdout}\nstderr:\n${stderr}`,
|
|
||||||
),
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
resolve();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-darwin-arm64",
|
"name": "@lancedb/lancedb-darwin-arm64",
|
||||||
"version": "0.28.0-beta.9",
|
"version": "0.28.0-beta.7",
|
||||||
"os": ["darwin"],
|
"os": ["darwin"],
|
||||||
"cpu": ["arm64"],
|
"cpu": ["arm64"],
|
||||||
"main": "lancedb.darwin-arm64.node",
|
"main": "lancedb.darwin-arm64.node",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-linux-arm64-gnu",
|
"name": "@lancedb/lancedb-linux-arm64-gnu",
|
||||||
"version": "0.28.0-beta.9",
|
"version": "0.28.0-beta.7",
|
||||||
"os": ["linux"],
|
"os": ["linux"],
|
||||||
"cpu": ["arm64"],
|
"cpu": ["arm64"],
|
||||||
"main": "lancedb.linux-arm64-gnu.node",
|
"main": "lancedb.linux-arm64-gnu.node",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-linux-arm64-musl",
|
"name": "@lancedb/lancedb-linux-arm64-musl",
|
||||||
"version": "0.28.0-beta.9",
|
"version": "0.28.0-beta.7",
|
||||||
"os": ["linux"],
|
"os": ["linux"],
|
||||||
"cpu": ["arm64"],
|
"cpu": ["arm64"],
|
||||||
"main": "lancedb.linux-arm64-musl.node",
|
"main": "lancedb.linux-arm64-musl.node",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-linux-x64-gnu",
|
"name": "@lancedb/lancedb-linux-x64-gnu",
|
||||||
"version": "0.28.0-beta.9",
|
"version": "0.28.0-beta.7",
|
||||||
"os": ["linux"],
|
"os": ["linux"],
|
||||||
"cpu": ["x64"],
|
"cpu": ["x64"],
|
||||||
"main": "lancedb.linux-x64-gnu.node",
|
"main": "lancedb.linux-x64-gnu.node",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-linux-x64-musl",
|
"name": "@lancedb/lancedb-linux-x64-musl",
|
||||||
"version": "0.28.0-beta.9",
|
"version": "0.28.0-beta.7",
|
||||||
"os": ["linux"],
|
"os": ["linux"],
|
||||||
"cpu": ["x64"],
|
"cpu": ["x64"],
|
||||||
"main": "lancedb.linux-x64-musl.node",
|
"main": "lancedb.linux-x64-musl.node",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-win32-arm64-msvc",
|
"name": "@lancedb/lancedb-win32-arm64-msvc",
|
||||||
"version": "0.28.0-beta.9",
|
"version": "0.28.0-beta.7",
|
||||||
"os": [
|
"os": [
|
||||||
"win32"
|
"win32"
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-win32-x64-msvc",
|
"name": "@lancedb/lancedb-win32-x64-msvc",
|
||||||
"version": "0.28.0-beta.9",
|
"version": "0.28.0-beta.7",
|
||||||
"os": ["win32"],
|
"os": ["win32"],
|
||||||
"cpu": ["x64"],
|
"cpu": ["x64"],
|
||||||
"main": "lancedb.win32-x64-msvc.node",
|
"main": "lancedb.win32-x64-msvc.node",
|
||||||
|
|||||||
Generated
+2
-2
@@ -1,12 +1,12 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb",
|
"name": "@lancedb/lancedb",
|
||||||
"version": "0.28.0-beta.9",
|
"version": "0.28.0-beta.7",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "@lancedb/lancedb",
|
"name": "@lancedb/lancedb",
|
||||||
"version": "0.28.0-beta.9",
|
"version": "0.28.0-beta.7",
|
||||||
"cpu": [
|
"cpu": [
|
||||||
"x64",
|
"x64",
|
||||||
"arm64"
|
"arm64"
|
||||||
|
|||||||
+1
-1
@@ -11,7 +11,7 @@
|
|||||||
"ann"
|
"ann"
|
||||||
],
|
],
|
||||||
"private": false,
|
"private": false,
|
||||||
"version": "0.28.0-beta.9",
|
"version": "0.28.0-beta.7",
|
||||||
"main": "dist/index.js",
|
"main": "dist/index.js",
|
||||||
"exports": {
|
"exports": {
|
||||||
".": "./dist/index.js",
|
".": "./dist/index.js",
|
||||||
|
|||||||
@@ -18,7 +18,6 @@ type RerankHybridFn = ThreadsafeFunction<
|
|||||||
RerankHybridCallbackArgs,
|
RerankHybridCallbackArgs,
|
||||||
Status,
|
Status,
|
||||||
false,
|
false,
|
||||||
true,
|
|
||||||
>;
|
>;
|
||||||
|
|
||||||
/// Reranker implementation that "wraps" a NodeJS Reranker implementation.
|
/// Reranker implementation that "wraps" a NodeJS Reranker implementation.
|
||||||
@@ -33,10 +32,7 @@ impl Reranker {
|
|||||||
pub fn new(
|
pub fn new(
|
||||||
rerank_hybrid: Function<RerankHybridCallbackArgs, Promise<Buffer>>,
|
rerank_hybrid: Function<RerankHybridCallbackArgs, Promise<Buffer>>,
|
||||||
) -> napi::Result<Self> {
|
) -> napi::Result<Self> {
|
||||||
let rerank_hybrid = rerank_hybrid
|
let rerank_hybrid = rerank_hybrid.build_threadsafe_function().build()?;
|
||||||
.build_threadsafe_function()
|
|
||||||
.weak::<true>()
|
|
||||||
.build()?;
|
|
||||||
Ok(Self { rerank_hybrid })
|
Ok(Self { rerank_hybrid })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
[tool.bumpversion]
|
[tool.bumpversion]
|
||||||
current_version = "0.31.0-beta.10"
|
current_version = "0.31.0-beta.7"
|
||||||
parse = """(?x)
|
parse = """(?x)
|
||||||
(?P<major>0|[1-9]\\d*)\\.
|
(?P<major>0|[1-9]\\d*)\\.
|
||||||
(?P<minor>0|[1-9]\\d*)\\.
|
(?P<minor>0|[1-9]\\d*)\\.
|
||||||
|
|||||||
+1
-2
@@ -1,7 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "lancedb-python"
|
name = "lancedb-python"
|
||||||
version = "0.31.0-beta.10"
|
version = "0.31.0-beta.7"
|
||||||
publish = false
|
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
description = "Python bindings for LanceDB"
|
description = "Python bindings for LanceDB"
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
|
|||||||
@@ -183,6 +183,7 @@
|
|||||||
| stack-data | 0.6.3 | MIT License | http://github.com/alexmojaki/stack_data |
|
| stack-data | 0.6.3 | MIT License | http://github.com/alexmojaki/stack_data |
|
||||||
| sympy | 1.14.0 | BSD License | https://sympy.org |
|
| sympy | 1.14.0 | BSD License | https://sympy.org |
|
||||||
| tabulate | 0.9.0 | MIT License | https://github.com/astanin/python-tabulate |
|
| tabulate | 0.9.0 | MIT License | https://github.com/astanin/python-tabulate |
|
||||||
|
| tantivy | 0.25.1 | UNKNOWN | UNKNOWN |
|
||||||
| threadpoolctl | 3.6.0 | BSD License | https://github.com/joblib/threadpoolctl |
|
| threadpoolctl | 3.6.0 | BSD License | https://github.com/joblib/threadpoolctl |
|
||||||
| timm | 1.0.24 | Apache Software License | https://github.com/huggingface/pytorch-image-models |
|
| timm | 1.0.24 | Apache Software License | https://github.com/huggingface/pytorch-image-models |
|
||||||
| tinycss2 | 1.4.0 | BSD License | https://www.courtbouillon.org/tinycss2 |
|
| tinycss2 | 1.4.0 | BSD License | https://www.courtbouillon.org/tinycss2 |
|
||||||
|
|||||||
@@ -57,6 +57,7 @@ tests = [
|
|||||||
"duckdb>=0.9.0",
|
"duckdb>=0.9.0",
|
||||||
"pytz>=2023.3",
|
"pytz>=2023.3",
|
||||||
"polars>=0.19, <=1.3.0",
|
"polars>=0.19, <=1.3.0",
|
||||||
|
"tantivy>=0.20.0",
|
||||||
"pyarrow-stubs>=16.0",
|
"pyarrow-stubs>=16.0",
|
||||||
"pylance>=5.0.0b5",
|
"pylance>=5.0.0b5",
|
||||||
"requests>=2.31.0",
|
"requests>=2.31.0",
|
||||||
|
|||||||
@@ -0,0 +1,201 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||||
|
|
||||||
|
"""Full text search index using tantivy-py"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
from typing import List, Tuple, Optional
|
||||||
|
|
||||||
|
import pyarrow as pa
|
||||||
|
|
||||||
|
try:
|
||||||
|
import tantivy
|
||||||
|
except ImportError:
|
||||||
|
raise ImportError(
|
||||||
|
"Please install tantivy-py `pip install tantivy` to use the full text search feature." # noqa: E501
|
||||||
|
)
|
||||||
|
|
||||||
|
from .table import LanceTable
|
||||||
|
|
||||||
|
|
||||||
|
def create_index(
|
||||||
|
index_path: str,
|
||||||
|
text_fields: List[str],
|
||||||
|
ordering_fields: Optional[List[str]] = None,
|
||||||
|
tokenizer_name: str = "default",
|
||||||
|
) -> tantivy.Index:
|
||||||
|
"""
|
||||||
|
Create a new Index (not populated)
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
index_path : str
|
||||||
|
Path to the index directory
|
||||||
|
text_fields : List[str]
|
||||||
|
List of text fields to index
|
||||||
|
ordering_fields: List[str]
|
||||||
|
List of unsigned type fields to order by at search time
|
||||||
|
tokenizer_name : str, default "default"
|
||||||
|
The tokenizer to use
|
||||||
|
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
index : tantivy.Index
|
||||||
|
The index object (not yet populated)
|
||||||
|
"""
|
||||||
|
if ordering_fields is None:
|
||||||
|
ordering_fields = []
|
||||||
|
# Declaring our schema.
|
||||||
|
schema_builder = tantivy.SchemaBuilder()
|
||||||
|
# special field that we'll populate with row_id
|
||||||
|
schema_builder.add_integer_field("doc_id", stored=True)
|
||||||
|
# data fields
|
||||||
|
for name in text_fields:
|
||||||
|
schema_builder.add_text_field(name, stored=True, tokenizer_name=tokenizer_name)
|
||||||
|
if ordering_fields:
|
||||||
|
for name in ordering_fields:
|
||||||
|
schema_builder.add_unsigned_field(name, fast=True)
|
||||||
|
schema = schema_builder.build()
|
||||||
|
os.makedirs(index_path, exist_ok=True)
|
||||||
|
index = tantivy.Index(schema, path=index_path)
|
||||||
|
return index
|
||||||
|
|
||||||
|
|
||||||
|
def populate_index(
|
||||||
|
index: tantivy.Index,
|
||||||
|
table: LanceTable,
|
||||||
|
fields: List[str],
|
||||||
|
writer_heap_size: Optional[int] = None,
|
||||||
|
ordering_fields: Optional[List[str]] = None,
|
||||||
|
) -> int:
|
||||||
|
"""
|
||||||
|
Populate an index with data from a LanceTable
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
index : tantivy.Index
|
||||||
|
The index object
|
||||||
|
table : LanceTable
|
||||||
|
The table to index
|
||||||
|
fields : List[str]
|
||||||
|
List of fields to index
|
||||||
|
writer_heap_size : int
|
||||||
|
The writer heap size in bytes, defaults to 1GB
|
||||||
|
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
int
|
||||||
|
The number of rows indexed
|
||||||
|
"""
|
||||||
|
if ordering_fields is None:
|
||||||
|
ordering_fields = []
|
||||||
|
writer_heap_size = writer_heap_size or 1024 * 1024 * 1024
|
||||||
|
# first check the fields exist and are string or large string type
|
||||||
|
nested = []
|
||||||
|
|
||||||
|
for name in fields:
|
||||||
|
try:
|
||||||
|
f = table.schema.field(name) # raises KeyError if not found
|
||||||
|
except KeyError:
|
||||||
|
f = resolve_path(table.schema, name)
|
||||||
|
nested.append(name)
|
||||||
|
|
||||||
|
if not pa.types.is_string(f.type) and not pa.types.is_large_string(f.type):
|
||||||
|
raise TypeError(f"Field {name} is not a string type")
|
||||||
|
|
||||||
|
# create a tantivy writer
|
||||||
|
writer = index.writer(heap_size=writer_heap_size)
|
||||||
|
# write data into index
|
||||||
|
dataset = table.to_lance()
|
||||||
|
row_id = 0
|
||||||
|
|
||||||
|
max_nested_level = 0
|
||||||
|
if len(nested) > 0:
|
||||||
|
max_nested_level = max([len(name.split(".")) for name in nested])
|
||||||
|
|
||||||
|
for b in dataset.to_batches(columns=fields + ordering_fields):
|
||||||
|
if max_nested_level > 0:
|
||||||
|
b = pa.Table.from_batches([b])
|
||||||
|
for _ in range(max_nested_level - 1):
|
||||||
|
b = b.flatten()
|
||||||
|
for i in range(b.num_rows):
|
||||||
|
doc = tantivy.Document()
|
||||||
|
for name in fields:
|
||||||
|
value = b[name][i].as_py()
|
||||||
|
if value is not None:
|
||||||
|
doc.add_text(name, value)
|
||||||
|
for name in ordering_fields:
|
||||||
|
value = b[name][i].as_py()
|
||||||
|
if value is not None:
|
||||||
|
doc.add_unsigned(name, value)
|
||||||
|
if not doc.is_empty:
|
||||||
|
doc.add_integer("doc_id", row_id)
|
||||||
|
writer.add_document(doc)
|
||||||
|
row_id += 1
|
||||||
|
# commit changes
|
||||||
|
writer.commit()
|
||||||
|
return row_id
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_path(schema, field_name: str) -> pa.Field:
|
||||||
|
"""
|
||||||
|
Resolve a nested field path to a list of field names
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
field_name : str
|
||||||
|
The field name to resolve
|
||||||
|
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
List[str]
|
||||||
|
The resolved path
|
||||||
|
"""
|
||||||
|
path = field_name.split(".")
|
||||||
|
field = schema.field(path.pop(0))
|
||||||
|
for segment in path:
|
||||||
|
if pa.types.is_struct(field.type):
|
||||||
|
field = field.type.field(segment)
|
||||||
|
else:
|
||||||
|
raise KeyError(f"field {field_name} not found in schema {schema}")
|
||||||
|
return field
|
||||||
|
|
||||||
|
|
||||||
|
def search_index(
|
||||||
|
index: tantivy.Index, query: str, limit: int = 10, ordering_field=None
|
||||||
|
) -> Tuple[Tuple[int], Tuple[float]]:
|
||||||
|
"""
|
||||||
|
Search an index for a query
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
index : tantivy.Index
|
||||||
|
The index object
|
||||||
|
query : str
|
||||||
|
The query string
|
||||||
|
limit : int
|
||||||
|
The maximum number of results to return
|
||||||
|
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
ids_and_score: list[tuple[int], tuple[float]]
|
||||||
|
A tuple of two tuples, the first containing the document ids
|
||||||
|
and the second containing the scores
|
||||||
|
"""
|
||||||
|
searcher = index.searcher()
|
||||||
|
query = index.parse_query(query)
|
||||||
|
# get top results
|
||||||
|
if ordering_field:
|
||||||
|
results = searcher.search(query, limit, order_by_field=ordering_field)
|
||||||
|
else:
|
||||||
|
results = searcher.search(query, limit)
|
||||||
|
if results.count == 0:
|
||||||
|
return tuple(), tuple()
|
||||||
|
return tuple(
|
||||||
|
zip(
|
||||||
|
*[
|
||||||
|
(searcher.doc(doc_address)["doc_id"][0], score)
|
||||||
|
for score, doc_address in results.hits
|
||||||
|
]
|
||||||
|
)
|
||||||
|
)
|
||||||
@@ -25,6 +25,7 @@ import deprecation
|
|||||||
import numpy as np
|
import numpy as np
|
||||||
import pyarrow as pa
|
import pyarrow as pa
|
||||||
import pyarrow.compute as pc
|
import pyarrow.compute as pc
|
||||||
|
import pyarrow.fs as pa_fs
|
||||||
import pydantic
|
import pydantic
|
||||||
|
|
||||||
from lancedb.pydantic import PYDANTIC_VERSION
|
from lancedb.pydantic import PYDANTIC_VERSION
|
||||||
@@ -1525,7 +1526,9 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
|
|||||||
return self._table._output_schema(self.to_query_object())
|
return self._table._output_schema(self.to_query_object())
|
||||||
|
|
||||||
def to_arrow(self, *, timeout: Optional[timedelta] = None) -> pa.Table:
|
def to_arrow(self, *, timeout: Optional[timedelta] = None) -> pa.Table:
|
||||||
self._table._ensure_no_legacy_fts_index()
|
path, fs, exist = self._table._get_fts_index_path()
|
||||||
|
if exist:
|
||||||
|
return self.tantivy_to_arrow()
|
||||||
|
|
||||||
query = self._query
|
query = self._query
|
||||||
if self._phrase_query:
|
if self._phrase_query:
|
||||||
@@ -1549,6 +1552,90 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
|
|||||||
):
|
):
|
||||||
raise NotImplementedError("to_batches on an FTS query")
|
raise NotImplementedError("to_batches on an FTS query")
|
||||||
|
|
||||||
|
def tantivy_to_arrow(self) -> pa.Table:
|
||||||
|
try:
|
||||||
|
import tantivy
|
||||||
|
except ImportError:
|
||||||
|
raise ImportError(
|
||||||
|
"Please install tantivy-py `pip install tantivy` to use the full text search feature." # noqa: E501
|
||||||
|
)
|
||||||
|
|
||||||
|
from .fts import search_index
|
||||||
|
|
||||||
|
# get the index path
|
||||||
|
path, fs, exist = self._table._get_fts_index_path()
|
||||||
|
|
||||||
|
# check if the index exist
|
||||||
|
if not exist:
|
||||||
|
raise FileNotFoundError(
|
||||||
|
"Fts index does not exist. "
|
||||||
|
"Please first call table.create_fts_index(['<field_names>']) to "
|
||||||
|
"create the fts index."
|
||||||
|
)
|
||||||
|
|
||||||
|
# Check that we are on local filesystem
|
||||||
|
if not isinstance(fs, pa_fs.LocalFileSystem):
|
||||||
|
raise NotImplementedError(
|
||||||
|
"Tantivy-based full text search "
|
||||||
|
"is only supported on the local filesystem"
|
||||||
|
)
|
||||||
|
# open the index
|
||||||
|
index = tantivy.Index.open(path)
|
||||||
|
# get the scores and doc ids
|
||||||
|
query = self._query
|
||||||
|
if self._phrase_query:
|
||||||
|
query = query.replace('"', "'")
|
||||||
|
query = f'"{query}"'
|
||||||
|
limit = self._limit if self._limit is not None else 10
|
||||||
|
row_ids, scores = search_index(
|
||||||
|
index, query, limit, ordering_field=self.ordering_field_name
|
||||||
|
)
|
||||||
|
if len(row_ids) == 0:
|
||||||
|
empty_schema = pa.schema([pa.field("_score", pa.float32())])
|
||||||
|
return pa.Table.from_batches([], schema=empty_schema)
|
||||||
|
scores = pa.array(scores)
|
||||||
|
output_tbl = self._table.to_lance().take(row_ids, columns=self._columns)
|
||||||
|
output_tbl = output_tbl.append_column("_score", scores)
|
||||||
|
# this needs to match vector search results which are uint64
|
||||||
|
row_ids = pa.array(row_ids, type=pa.uint64())
|
||||||
|
|
||||||
|
if self._where is not None:
|
||||||
|
tmp_name = "__lancedb__duckdb__indexer__"
|
||||||
|
output_tbl = output_tbl.append_column(
|
||||||
|
tmp_name, pa.array(range(len(output_tbl)))
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
# TODO would be great to have Substrait generate pyarrow compute
|
||||||
|
# expressions or conversely have pyarrow support SQL expressions
|
||||||
|
# using Substrait
|
||||||
|
import duckdb
|
||||||
|
|
||||||
|
indexer = duckdb.sql(
|
||||||
|
f"SELECT {tmp_name} FROM output_tbl WHERE {self._where}"
|
||||||
|
).to_arrow_table()[tmp_name]
|
||||||
|
output_tbl = output_tbl.take(indexer).drop([tmp_name])
|
||||||
|
row_ids = row_ids.take(indexer)
|
||||||
|
|
||||||
|
except ImportError:
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
import lance
|
||||||
|
|
||||||
|
# TODO Use "memory://" instead once that's supported
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
ds = lance.write_dataset(output_tbl, tmp)
|
||||||
|
output_tbl = ds.to_table(filter=self._where)
|
||||||
|
indexer = output_tbl[tmp_name]
|
||||||
|
row_ids = row_ids.take(indexer)
|
||||||
|
output_tbl = output_tbl.drop([tmp_name])
|
||||||
|
|
||||||
|
if self._with_row_id:
|
||||||
|
output_tbl = output_tbl.append_column("_rowid", row_ids)
|
||||||
|
|
||||||
|
if self._reranker is not None:
|
||||||
|
output_tbl = self._reranker.rerank_fts(self._query, output_tbl)
|
||||||
|
return output_tbl
|
||||||
|
|
||||||
def rerank(self, reranker: Reranker) -> LanceFtsQueryBuilder:
|
def rerank(self, reranker: Reranker) -> LanceFtsQueryBuilder:
|
||||||
"""Rerank the results using the specified reranker.
|
"""Rerank the results using the specified reranker.
|
||||||
|
|
||||||
|
|||||||
@@ -943,26 +943,29 @@ class Table(ABC):
|
|||||||
Parameters
|
Parameters
|
||||||
----------
|
----------
|
||||||
field_names: str or list of str
|
field_names: str or list of str
|
||||||
The name of the field to index. Native FTS indexes can only be
|
The name(s) of the field to index.
|
||||||
created on a single field at a time. To search over multiple text
|
If ``use_tantivy`` is False (default), only a single field name
|
||||||
fields, create a separate FTS index for each field.
|
(str) is supported. To index multiple fields, create a separate
|
||||||
|
FTS index for each field.
|
||||||
replace: bool, default False
|
replace: bool, default False
|
||||||
If True, replace the existing index if it exists. Note that this is
|
If True, replace the existing index if it exists. Note that this is
|
||||||
not yet an atomic operation; the index will be temporarily
|
not yet an atomic operation; the index will be temporarily
|
||||||
unavailable while the new index is being created.
|
unavailable while the new index is being created.
|
||||||
writer_heap_size: int, default 1GB
|
writer_heap_size: int, default 1GB
|
||||||
Deprecated legacy Tantivy parameter. Any value other than the
|
Only available with use_tantivy=True
|
||||||
default raises an error.
|
|
||||||
ordering_field_names:
|
ordering_field_names:
|
||||||
Deprecated legacy Tantivy parameter. Setting this raises an error.
|
A list of unsigned type fields to index to optionally order
|
||||||
|
results on at search time.
|
||||||
|
only available with use_tantivy=True
|
||||||
tokenizer_name: str, default "default"
|
tokenizer_name: str, default "default"
|
||||||
A compatibility alias for native tokenizer configs. Can be "raw",
|
The tokenizer to use for the index. Can be "raw", "default" or the 2 letter
|
||||||
"default" or the 2 letter language code followed by "_stem". So
|
language code followed by "_stem". So for english it would be "en_stem".
|
||||||
for english it would be "en_stem".
|
For available languages see: https://docs.rs/tantivy/latest/tantivy/tokenizer/enum.Language.html
|
||||||
use_tantivy: bool, default False
|
use_tantivy: bool, default False
|
||||||
Deprecated legacy Tantivy parameter. Setting this to True raises an
|
If True, use the legacy full-text search implementation based on tantivy.
|
||||||
error.
|
If False, use the new full-text search implementation based on lance-index.
|
||||||
with_position: bool, default False
|
with_position: bool, default False
|
||||||
|
Only available with use_tantivy=False
|
||||||
If False, do not store the positions of the terms in the text.
|
If False, do not store the positions of the terms in the text.
|
||||||
This can reduce the size of the index and improve indexing speed.
|
This can reduce the size of the index and improve indexing speed.
|
||||||
But it will raise an exception for phrase queries.
|
But it will raise an exception for phrase queries.
|
||||||
@@ -1743,16 +1746,6 @@ class Table(ABC):
|
|||||||
index_exists = fs.get_file_info(path).type != pa_fs.FileType.NotFound
|
index_exists = fs.get_file_info(path).type != pa_fs.FileType.NotFound
|
||||||
return (path, fs, index_exists)
|
return (path, fs, index_exists)
|
||||||
|
|
||||||
def _ensure_no_legacy_fts_index(self):
|
|
||||||
path, _, exists = self._get_fts_index_path()
|
|
||||||
if exists:
|
|
||||||
raise ValueError(
|
|
||||||
"Legacy Tantivy FTS index detected at "
|
|
||||||
f"{path}. Tantivy-based FTS has been removed. "
|
|
||||||
"Delete the legacy index and recreate it with "
|
|
||||||
"table.create_fts_index(...)."
|
|
||||||
)
|
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def uses_v2_manifest_paths(self) -> bool:
|
def uses_v2_manifest_paths(self) -> bool:
|
||||||
"""
|
"""
|
||||||
@@ -2412,63 +2405,84 @@ class LanceTable(Table):
|
|||||||
prefix_only: bool = False,
|
prefix_only: bool = False,
|
||||||
name: Optional[str] = None,
|
name: Optional[str] = None,
|
||||||
):
|
):
|
||||||
self._ensure_no_legacy_fts_index()
|
if not use_tantivy:
|
||||||
|
if not isinstance(field_names, str):
|
||||||
|
raise ValueError(
|
||||||
|
"Native FTS indexes can only be created on a single field "
|
||||||
|
"at a time. To search over multiple text fields, create a "
|
||||||
|
"separate FTS index for each field."
|
||||||
|
)
|
||||||
|
|
||||||
if use_tantivy:
|
if tokenizer_name is None:
|
||||||
raise ValueError(
|
tokenizer_configs = {
|
||||||
"Tantivy-based FTS has been removed. "
|
"base_tokenizer": base_tokenizer,
|
||||||
"Remove use_tantivy and recreate the index with native FTS."
|
"language": language,
|
||||||
|
"with_position": with_position,
|
||||||
|
"max_token_length": max_token_length,
|
||||||
|
"lower_case": lower_case,
|
||||||
|
"stem": stem,
|
||||||
|
"remove_stop_words": remove_stop_words,
|
||||||
|
"ascii_folding": ascii_folding,
|
||||||
|
"ngram_min_length": ngram_min_length,
|
||||||
|
"ngram_max_length": ngram_max_length,
|
||||||
|
"prefix_only": prefix_only,
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
tokenizer_configs = self.infer_tokenizer_configs(tokenizer_name)
|
||||||
|
|
||||||
|
config = FTS(
|
||||||
|
**tokenizer_configs,
|
||||||
)
|
)
|
||||||
if ordering_field_names is not None:
|
|
||||||
raise ValueError(
|
# delete the existing legacy index if it exists
|
||||||
"ordering_field_names was only supported by the removed "
|
if replace:
|
||||||
"Tantivy-based FTS implementation."
|
path, fs, exist = self._get_fts_index_path()
|
||||||
|
if exist:
|
||||||
|
fs.delete_dir(path)
|
||||||
|
|
||||||
|
LOOP.run(
|
||||||
|
self._table.create_index(
|
||||||
|
field_names,
|
||||||
|
replace=replace,
|
||||||
|
config=config,
|
||||||
|
name=name,
|
||||||
|
)
|
||||||
)
|
)
|
||||||
if writer_heap_size != 1024 * 1024 * 1024:
|
return
|
||||||
raise ValueError(
|
|
||||||
"writer_heap_size was only supported by the removed "
|
from .fts import create_index, populate_index
|
||||||
"Tantivy-based FTS implementation."
|
|
||||||
)
|
if isinstance(field_names, str):
|
||||||
if not isinstance(field_names, str):
|
field_names = [field_names]
|
||||||
raise ValueError(
|
|
||||||
"Native FTS indexes can only be created on a single field "
|
if isinstance(ordering_field_names, str):
|
||||||
"at a time. To search over multiple text fields, create a "
|
ordering_field_names = [ordering_field_names]
|
||||||
"separate FTS index for each field."
|
|
||||||
)
|
path, fs, exist = self._get_fts_index_path()
|
||||||
if "." in field_names:
|
if exist:
|
||||||
raise ValueError(
|
if not replace:
|
||||||
"Native FTS indexes can only be created on top-level fields. "
|
raise ValueError("Index already exists. Use replace=True to overwrite.")
|
||||||
f"Received nested field path: {field_names!r}."
|
fs.delete_dir(path)
|
||||||
|
|
||||||
|
if not isinstance(fs, pa_fs.LocalFileSystem):
|
||||||
|
raise NotImplementedError(
|
||||||
|
"Full-text search is only supported on the local filesystem"
|
||||||
)
|
)
|
||||||
|
|
||||||
if tokenizer_name is None:
|
if tokenizer_name is None:
|
||||||
tokenizer_configs = {
|
tokenizer_name = "default"
|
||||||
"base_tokenizer": base_tokenizer,
|
index = create_index(
|
||||||
"language": language,
|
path,
|
||||||
"with_position": with_position,
|
field_names,
|
||||||
"max_token_length": max_token_length,
|
ordering_fields=ordering_field_names,
|
||||||
"lower_case": lower_case,
|
tokenizer_name=tokenizer_name,
|
||||||
"stem": stem,
|
|
||||||
"remove_stop_words": remove_stop_words,
|
|
||||||
"ascii_folding": ascii_folding,
|
|
||||||
"ngram_min_length": ngram_min_length,
|
|
||||||
"ngram_max_length": ngram_max_length,
|
|
||||||
"prefix_only": prefix_only,
|
|
||||||
}
|
|
||||||
else:
|
|
||||||
tokenizer_configs = self.infer_tokenizer_configs(tokenizer_name)
|
|
||||||
|
|
||||||
config = FTS(
|
|
||||||
**tokenizer_configs,
|
|
||||||
)
|
)
|
||||||
|
populate_index(
|
||||||
LOOP.run(
|
index,
|
||||||
self._table.create_index(
|
self,
|
||||||
field_names,
|
field_names,
|
||||||
replace=replace,
|
ordering_fields=ordering_field_names,
|
||||||
config=config,
|
writer_heap_size=writer_heap_size,
|
||||||
name=name,
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|||||||
@@ -180,7 +180,7 @@ def test_fts_fuzzy_query():
|
|||||||
),
|
),
|
||||||
mode="overwrite",
|
mode="overwrite",
|
||||||
)
|
)
|
||||||
table.create_fts_index("text", replace=True)
|
table.create_fts_index("text", use_tantivy=False, replace=True)
|
||||||
|
|
||||||
results = table.search(MatchQuery("foo", "text", fuzziness=1)).to_pandas()
|
results = table.search(MatchQuery("foo", "text", fuzziness=1)).to_pandas()
|
||||||
assert len(results) == 4
|
assert len(results) == 4
|
||||||
@@ -230,7 +230,7 @@ def test_fts_boost_query():
|
|||||||
),
|
),
|
||||||
mode="overwrite",
|
mode="overwrite",
|
||||||
)
|
)
|
||||||
table.create_fts_index("desc", replace=True)
|
table.create_fts_index("desc", use_tantivy=False, replace=True)
|
||||||
|
|
||||||
results = table.search(
|
results = table.search(
|
||||||
BoostQuery(
|
BoostQuery(
|
||||||
@@ -265,7 +265,7 @@ def test_fts_boolean_query(tmp_path):
|
|||||||
],
|
],
|
||||||
mode="overwrite",
|
mode="overwrite",
|
||||||
)
|
)
|
||||||
table.create_fts_index("text", replace=True)
|
table.create_fts_index("text", use_tantivy=False, replace=True)
|
||||||
|
|
||||||
# SHOULD
|
# SHOULD
|
||||||
results = table.search(
|
results = table.search(
|
||||||
@@ -319,7 +319,9 @@ def test_fts_native():
|
|||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
table.create_fts_index("text")
|
# passing `use_tantivy=False` to use lance FTS index
|
||||||
|
# `use_tantivy=True` by default
|
||||||
|
table.create_fts_index("text", use_tantivy=False)
|
||||||
table.search("puppy").limit(10).select(["text"]).to_list()
|
table.search("puppy").limit(10).select(["text"]).to_list()
|
||||||
# [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}]
|
# [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}]
|
||||||
# ...
|
# ...
|
||||||
@@ -330,6 +332,7 @@ def test_fts_native():
|
|||||||
# --8<-- [start:fts_config_folding]
|
# --8<-- [start:fts_config_folding]
|
||||||
table.create_fts_index(
|
table.create_fts_index(
|
||||||
"text",
|
"text",
|
||||||
|
use_tantivy=False,
|
||||||
language="French",
|
language="French",
|
||||||
stem=True,
|
stem=True,
|
||||||
ascii_folding=True,
|
ascii_folding=True,
|
||||||
@@ -343,7 +346,7 @@ def test_fts_native():
|
|||||||
table.search("puppy").limit(10).where("text='foo'", prefilter=False).to_list()
|
table.search("puppy").limit(10).where("text='foo'", prefilter=False).to_list()
|
||||||
# --8<-- [end:fts_postfiltering]
|
# --8<-- [end:fts_postfiltering]
|
||||||
# --8<-- [start:fts_with_position]
|
# --8<-- [start:fts_with_position]
|
||||||
table.create_fts_index("text", with_position=True, replace=True)
|
table.create_fts_index("text", use_tantivy=False, with_position=True, replace=True)
|
||||||
# --8<-- [end:fts_with_position]
|
# --8<-- [end:fts_with_position]
|
||||||
# --8<-- [start:fts_incremental_index]
|
# --8<-- [start:fts_incremental_index]
|
||||||
table.add([{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}])
|
table.add([{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}])
|
||||||
|
|||||||
@@ -15,7 +15,8 @@ import pytest
|
|||||||
from lancedb.pydantic import LanceModel, Vector
|
from lancedb.pydantic import LanceModel, Vector
|
||||||
|
|
||||||
|
|
||||||
def test_basic(tmp_path):
|
@pytest.mark.parametrize("use_tantivy", [True, False])
|
||||||
|
def test_basic(tmp_path, use_tantivy):
|
||||||
db = lancedb.connect(tmp_path)
|
db = lancedb.connect(tmp_path)
|
||||||
|
|
||||||
assert db.uri == str(tmp_path)
|
assert db.uri == str(tmp_path)
|
||||||
@@ -48,7 +49,7 @@ def test_basic(tmp_path):
|
|||||||
assert len(rs) == 1
|
assert len(rs) == 1
|
||||||
assert rs["item"].iloc[0] == "foo"
|
assert rs["item"].iloc[0] == "foo"
|
||||||
|
|
||||||
table.create_fts_index("item")
|
table.create_fts_index("item", use_tantivy=use_tantivy)
|
||||||
rs = table.search("bar", query_type="fts").to_pandas()
|
rs = table.search("bar", query_type="fts").to_pandas()
|
||||||
assert len(rs) == 1
|
assert len(rs) == 1
|
||||||
assert rs["item"].iloc[0] == "bar"
|
assert rs["item"].iloc[0] == "bar"
|
||||||
|
|||||||
+158
-100
@@ -36,6 +36,9 @@ import pytest
|
|||||||
import pytest_asyncio
|
import pytest_asyncio
|
||||||
from utils import exception_output
|
from utils import exception_output
|
||||||
|
|
||||||
|
pytest.importorskip("lancedb.fts")
|
||||||
|
tantivy = pytest.importorskip("tantivy")
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def table(tmp_path) -> ldb.table.LanceTable:
|
def table(tmp_path) -> ldb.table.LanceTable:
|
||||||
@@ -141,53 +144,58 @@ async def async_table(tmp_path) -> ldb.table.AsyncTable:
|
|||||||
return table
|
return table
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
def test_create_index(tmp_path):
|
||||||
("kwargs", "match"),
|
index = ldb.fts.create_index(str(tmp_path / "index"), ["text"])
|
||||||
[
|
assert isinstance(index, tantivy.Index)
|
||||||
(
|
assert os.path.exists(str(tmp_path / "index"))
|
||||||
{"use_tantivy": True},
|
|
||||||
"Tantivy-based FTS has been removed",
|
|
||||||
),
|
|
||||||
(
|
|
||||||
{"ordering_field_names": ["count"]},
|
|
||||||
"ordering_field_names was only supported",
|
|
||||||
),
|
|
||||||
(
|
|
||||||
{"writer_heap_size": 128},
|
|
||||||
"writer_heap_size was only supported",
|
|
||||||
),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def test_reject_removed_tantivy_parameters(table, kwargs, match):
|
|
||||||
with pytest.raises(ValueError, match=match):
|
|
||||||
table.create_fts_index("text", **kwargs)
|
|
||||||
|
|
||||||
|
|
||||||
def test_reject_legacy_tantivy_index(table):
|
def test_create_index_with_stemming(tmp_path, table):
|
||||||
path, _, _ = table._get_fts_index_path()
|
index = ldb.fts.create_index(
|
||||||
os.makedirs(path, exist_ok=True)
|
str(tmp_path / "index"), ["text"], tokenizer_name="en_stem"
|
||||||
|
)
|
||||||
|
assert isinstance(index, tantivy.Index)
|
||||||
|
assert os.path.exists(str(tmp_path / "index"))
|
||||||
|
|
||||||
with pytest.raises(ValueError, match="Legacy Tantivy FTS index detected"):
|
# Check stemming by running tokenizer on non empty table
|
||||||
table.search("puppy").limit(5).to_list()
|
table.create_fts_index("text", tokenizer_name="en_stem", use_tantivy=True)
|
||||||
|
|
||||||
with pytest.raises(ValueError, match="Legacy Tantivy FTS index detected"):
|
|
||||||
table.create_fts_index("text")
|
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("use_tantivy", [True, False])
|
||||||
@pytest.mark.parametrize("with_position", [True, False])
|
@pytest.mark.parametrize("with_position", [True, False])
|
||||||
def test_create_inverted_index(table, with_position):
|
def test_create_inverted_index(table, use_tantivy, with_position):
|
||||||
|
if use_tantivy and not with_position:
|
||||||
|
pytest.skip("we don't support building a tantivy index without position")
|
||||||
table.create_fts_index(
|
table.create_fts_index(
|
||||||
"text",
|
"text",
|
||||||
|
use_tantivy=use_tantivy,
|
||||||
with_position=with_position,
|
with_position=with_position,
|
||||||
name="custom_fts_index",
|
name="custom_fts_index",
|
||||||
)
|
)
|
||||||
indices = table.list_indices()
|
if not use_tantivy:
|
||||||
fts_indices = [i for i in indices if i.index_type == "FTS"]
|
indices = table.list_indices()
|
||||||
assert any(i.name == "custom_fts_index" for i in fts_indices)
|
fts_indices = [i for i in indices if i.index_type == "FTS"]
|
||||||
|
assert any(i.name == "custom_fts_index" for i in fts_indices)
|
||||||
|
|
||||||
|
|
||||||
def test_search_fts(table):
|
def test_populate_index(tmp_path, table):
|
||||||
table.create_fts_index("text")
|
index = ldb.fts.create_index(str(tmp_path / "index"), ["text"])
|
||||||
|
assert ldb.fts.populate_index(index, table, ["text"]) == len(table)
|
||||||
|
|
||||||
|
|
||||||
|
def test_search_index(tmp_path, table):
|
||||||
|
index = ldb.fts.create_index(str(tmp_path / "index"), ["text"])
|
||||||
|
ldb.fts.populate_index(index, table, ["text"])
|
||||||
|
index.reload()
|
||||||
|
results = ldb.fts.search_index(index, query="puppy", limit=5)
|
||||||
|
assert len(results) == 2
|
||||||
|
assert len(results[0]) == 5 # row_ids
|
||||||
|
assert len(results[1]) == 5 # _score
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("use_tantivy", [True, False])
|
||||||
|
def test_search_fts(table, use_tantivy):
|
||||||
|
table.create_fts_index("text", use_tantivy=use_tantivy)
|
||||||
results = table.search("puppy").select(["id", "text"]).limit(5).to_list()
|
results = table.search("puppy").select(["id", "text"]).limit(5).to_list()
|
||||||
assert len(results) == 5
|
assert len(results) == 5
|
||||||
assert len(results[0]) == 3 # id, text, _score
|
assert len(results[0]) == 3 # id, text, _score
|
||||||
@@ -196,52 +204,53 @@ def test_search_fts(table):
|
|||||||
results = table.search("puppy").select(["id", "text"]).to_list()
|
results = table.search("puppy").select(["id", "text"]).to_list()
|
||||||
assert len(results) == 10
|
assert len(results) == 10
|
||||||
|
|
||||||
# Test with a query
|
if not use_tantivy:
|
||||||
results = (
|
# Test with a query
|
||||||
table.search(MatchQuery("puppy", "text"))
|
results = (
|
||||||
.select(["id", "text"])
|
table.search(MatchQuery("puppy", "text"))
|
||||||
.limit(5)
|
.select(["id", "text"])
|
||||||
.to_list()
|
.limit(5)
|
||||||
)
|
.to_list()
|
||||||
assert len(results) == 5
|
|
||||||
|
|
||||||
# Test boost query
|
|
||||||
results = (
|
|
||||||
table.search(
|
|
||||||
BoostQuery(
|
|
||||||
MatchQuery("puppy", "text"),
|
|
||||||
MatchQuery("runs", "text"),
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
.select(["id", "text"])
|
assert len(results) == 5
|
||||||
.limit(5)
|
|
||||||
.to_list()
|
|
||||||
)
|
|
||||||
assert len(results) == 5
|
|
||||||
|
|
||||||
# Test multi match query
|
# Test boost query
|
||||||
table.create_fts_index("text2")
|
results = (
|
||||||
results = (
|
table.search(
|
||||||
table.search(MultiMatchQuery("puppy", ["text", "text2"]))
|
BoostQuery(
|
||||||
.select(["id", "text"])
|
MatchQuery("puppy", "text"),
|
||||||
.limit(5)
|
MatchQuery("runs", "text"),
|
||||||
.to_list()
|
)
|
||||||
)
|
)
|
||||||
assert len(results) == 5
|
.select(["id", "text"])
|
||||||
assert len(results[0]) == 3 # id, text, _score
|
.limit(5)
|
||||||
|
.to_list()
|
||||||
|
)
|
||||||
|
assert len(results) == 5
|
||||||
|
|
||||||
# Test boolean query
|
# Test multi match query
|
||||||
results = (
|
table.create_fts_index("text2", use_tantivy=use_tantivy)
|
||||||
table.search(MatchQuery("puppy", "text") & MatchQuery("runs", "text"))
|
results = (
|
||||||
.select(["id", "text"])
|
table.search(MultiMatchQuery("puppy", ["text", "text2"]))
|
||||||
.limit(5)
|
.select(["id", "text"])
|
||||||
.to_list()
|
.limit(5)
|
||||||
)
|
.to_list()
|
||||||
assert len(results) == 5
|
)
|
||||||
assert len(results[0]) == 3 # id, text, _score
|
assert len(results) == 5
|
||||||
for r in results:
|
assert len(results[0]) == 3 # id, text, _score
|
||||||
assert "puppy" in r["text"]
|
|
||||||
assert "runs" in r["text"]
|
# Test boolean query
|
||||||
|
results = (
|
||||||
|
table.search(MatchQuery("puppy", "text") & MatchQuery("runs", "text"))
|
||||||
|
.select(["id", "text"])
|
||||||
|
.limit(5)
|
||||||
|
.to_list()
|
||||||
|
)
|
||||||
|
assert len(results) == 5
|
||||||
|
assert len(results[0]) == 3 # id, text, _score
|
||||||
|
for r in results:
|
||||||
|
assert "puppy" in r["text"]
|
||||||
|
assert "runs" in r["text"]
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
@@ -309,13 +318,13 @@ async def test_fts_select_async(async_table):
|
|||||||
|
|
||||||
|
|
||||||
def test_search_fts_phrase_query(table):
|
def test_search_fts_phrase_query(table):
|
||||||
table.create_fts_index("text", with_position=False)
|
table.create_fts_index("text", use_tantivy=False, with_position=False)
|
||||||
try:
|
try:
|
||||||
phrase_results = table.search('"puppy runs"').limit(100).to_list()
|
phrase_results = table.search('"puppy runs"').limit(100).to_list()
|
||||||
assert False
|
assert False
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
table.create_fts_index("text", with_position=True, replace=True)
|
table.create_fts_index("text", use_tantivy=False, with_position=True, replace=True)
|
||||||
results = table.search("puppy").limit(100).to_list()
|
results = table.search("puppy").limit(100).to_list()
|
||||||
|
|
||||||
# Test with quotation marks
|
# Test with quotation marks
|
||||||
@@ -366,8 +375,8 @@ async def test_search_fts_phrase_query_async(async_table):
|
|||||||
|
|
||||||
|
|
||||||
def test_search_fts_specify_column(table):
|
def test_search_fts_specify_column(table):
|
||||||
table.create_fts_index("text")
|
table.create_fts_index("text", use_tantivy=False)
|
||||||
table.create_fts_index("text2")
|
table.create_fts_index("text2", use_tantivy=False)
|
||||||
|
|
||||||
results = table.search("puppy", fts_columns="text").limit(5).to_list()
|
results = table.search("puppy", fts_columns="text").limit(5).to_list()
|
||||||
assert len(results) == 5
|
assert len(results) == 5
|
||||||
@@ -461,8 +470,42 @@ async def test_search_fts_specify_column_async(async_table):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
def test_create_index_from_table(tmp_path, table):
|
def test_search_ordering_field_index_table(tmp_path, table):
|
||||||
table.create_fts_index("text")
|
table.create_fts_index("text", ordering_field_names=["count"], use_tantivy=True)
|
||||||
|
rows = (
|
||||||
|
table.search("puppy", ordering_field_name="count")
|
||||||
|
.limit(20)
|
||||||
|
.select(["text", "count"])
|
||||||
|
.to_list()
|
||||||
|
)
|
||||||
|
for r in rows:
|
||||||
|
assert "puppy" in r["text"]
|
||||||
|
assert sorted(rows, key=lambda x: x["count"], reverse=True) == rows
|
||||||
|
|
||||||
|
|
||||||
|
def test_search_ordering_field_index(tmp_path, table):
|
||||||
|
index = ldb.fts.create_index(
|
||||||
|
str(tmp_path / "index"), ["text"], ordering_fields=["count"]
|
||||||
|
)
|
||||||
|
|
||||||
|
ldb.fts.populate_index(index, table, ["text"], ordering_fields=["count"])
|
||||||
|
index.reload()
|
||||||
|
results = ldb.fts.search_index(
|
||||||
|
index, query="puppy", limit=5, ordering_field="count"
|
||||||
|
)
|
||||||
|
assert len(results) == 2
|
||||||
|
assert len(results[0]) == 5 # row_ids
|
||||||
|
assert len(results[1]) == 5 # _distance
|
||||||
|
rows = table.to_lance().take(results[0]).to_pylist()
|
||||||
|
|
||||||
|
for r in rows:
|
||||||
|
assert "puppy" in r["text"]
|
||||||
|
assert sorted(rows, key=lambda x: x["count"], reverse=True) == rows
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("use_tantivy", [True, False])
|
||||||
|
def test_create_index_from_table(tmp_path, table, use_tantivy):
|
||||||
|
table.create_fts_index("text", use_tantivy=use_tantivy)
|
||||||
df = table.search("puppy").limit(5).select(["text"]).to_pandas()
|
df = table.search("puppy").limit(5).select(["text"]).to_pandas()
|
||||||
assert len(df) <= 5
|
assert len(df) <= 5
|
||||||
assert "text" in df.columns
|
assert "text" in df.columns
|
||||||
@@ -482,24 +525,36 @@ def test_create_index_from_table(tmp_path, table):
|
|||||||
)
|
)
|
||||||
|
|
||||||
with pytest.raises(Exception, match="already exists"):
|
with pytest.raises(Exception, match="already exists"):
|
||||||
table.create_fts_index("text")
|
table.create_fts_index("text", use_tantivy=use_tantivy)
|
||||||
|
|
||||||
table.create_fts_index("text", replace=True)
|
table.create_fts_index("text", replace=True, use_tantivy=use_tantivy)
|
||||||
assert len(table.search("gorilla").limit(1).to_pandas()) == 1
|
assert len(table.search("gorilla").limit(1).to_pandas()) == 1
|
||||||
|
|
||||||
|
|
||||||
def test_create_index_multiple_columns(tmp_path, table):
|
def test_create_index_multiple_columns(tmp_path, table):
|
||||||
with pytest.raises(ValueError, match="Native FTS indexes can only be created"):
|
table.create_fts_index(["text", "text2"], use_tantivy=True)
|
||||||
table.create_fts_index(["text", "text2"])
|
df = table.search("puppy").limit(5).to_pandas()
|
||||||
|
assert len(df) == 5
|
||||||
|
assert "text" in df.columns
|
||||||
|
assert "text2" in df.columns
|
||||||
|
|
||||||
|
|
||||||
|
def test_empty_rs(tmp_path, table, mocker):
|
||||||
|
table.create_fts_index(["text", "text2"], use_tantivy=True)
|
||||||
|
mocker.patch("lancedb.fts.search_index", return_value=([], []))
|
||||||
|
df = table.search("puppy").limit(5).to_pandas()
|
||||||
|
assert len(df) == 0
|
||||||
|
|
||||||
|
|
||||||
def test_nested_schema(tmp_path, table):
|
def test_nested_schema(tmp_path, table):
|
||||||
with pytest.raises(ValueError, match="top-level fields"):
|
table.create_fts_index("nested.text", use_tantivy=True)
|
||||||
table.create_fts_index("nested.text")
|
rs = table.search("puppy").limit(5).to_list()
|
||||||
|
assert len(rs) == 5
|
||||||
|
|
||||||
|
|
||||||
def test_search_index_with_filter(table):
|
@pytest.mark.parametrize("use_tantivy", [True, False])
|
||||||
table.create_fts_index("text")
|
def test_search_index_with_filter(table, use_tantivy):
|
||||||
|
table.create_fts_index("text", use_tantivy=use_tantivy)
|
||||||
orig_import = __import__
|
orig_import = __import__
|
||||||
|
|
||||||
def import_mock(name, *args):
|
def import_mock(name, *args):
|
||||||
@@ -529,7 +584,8 @@ def test_search_index_with_filter(table):
|
|||||||
assert r["_rowid"] is not None
|
assert r["_rowid"] is not None
|
||||||
|
|
||||||
|
|
||||||
def test_null_input(table):
|
@pytest.mark.parametrize("use_tantivy", [True, False])
|
||||||
|
def test_null_input(table, use_tantivy):
|
||||||
table.add(
|
table.add(
|
||||||
[
|
[
|
||||||
{
|
{
|
||||||
@@ -542,13 +598,14 @@ def test_null_input(table):
|
|||||||
}
|
}
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
table.create_fts_index("text")
|
table.create_fts_index("text", use_tantivy=use_tantivy)
|
||||||
|
|
||||||
|
|
||||||
def test_syntax(table):
|
def test_syntax(table):
|
||||||
# https://github.com/lancedb/lancedb/issues/769
|
# https://github.com/lancedb/lancedb/issues/769
|
||||||
table.create_fts_index("text")
|
table.create_fts_index("text", use_tantivy=True)
|
||||||
table.search("they could have been dogs OR").limit(10).to_list()
|
with pytest.raises(ValueError, match="Syntax Error"):
|
||||||
|
table.search("they could have been dogs OR").limit(10).to_list()
|
||||||
|
|
||||||
# these should work
|
# these should work
|
||||||
|
|
||||||
@@ -559,7 +616,6 @@ def test_syntax(table):
|
|||||||
).to_list()
|
).to_list()
|
||||||
|
|
||||||
# phrase queries
|
# phrase queries
|
||||||
table.create_fts_index("text", with_position=True, replace=True)
|
|
||||||
table.search("they could have been dogs OR cats").phrase_query().limit(10).to_list()
|
table.search("they could have been dogs OR cats").phrase_query().limit(10).to_list()
|
||||||
table.search('"they could have been dogs OR cats"').limit(10).to_list()
|
table.search('"they could have been dogs OR cats"').limit(10).to_list()
|
||||||
table.search('''"the cats OR dogs were not really 'pets' at all"''').limit(
|
table.search('''"the cats OR dogs were not really 'pets' at all"''').limit(
|
||||||
@@ -583,7 +639,7 @@ def test_language(mem_db: DBConnection):
|
|||||||
table = mem_db.create_table("test", data=data)
|
table = mem_db.create_table("test", data=data)
|
||||||
|
|
||||||
with pytest.raises(ValueError) as e:
|
with pytest.raises(ValueError) as e:
|
||||||
table.create_fts_index("text", language="klingon")
|
table.create_fts_index("text", use_tantivy=False, language="klingon")
|
||||||
|
|
||||||
assert exception_output(e) == (
|
assert exception_output(e) == (
|
||||||
"ValueError: LanceDB does not support the requested language: 'klingon'\n"
|
"ValueError: LanceDB does not support the requested language: 'klingon'\n"
|
||||||
@@ -594,6 +650,7 @@ def test_language(mem_db: DBConnection):
|
|||||||
|
|
||||||
table.create_fts_index(
|
table.create_fts_index(
|
||||||
"text",
|
"text",
|
||||||
|
use_tantivy=False,
|
||||||
language="French",
|
language="French",
|
||||||
stem=True,
|
stem=True,
|
||||||
ascii_folding=True,
|
ascii_folding=True,
|
||||||
@@ -633,7 +690,7 @@ def test_fts_on_list(mem_db: DBConnection):
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
table = mem_db.create_table("test", data=data)
|
table = mem_db.create_table("test", data=data)
|
||||||
table.create_fts_index("text", with_position=True)
|
table.create_fts_index("text", use_tantivy=False, with_position=True)
|
||||||
|
|
||||||
res = table.search("lance").limit(5).to_list()
|
res = table.search("lance").limit(5).to_list()
|
||||||
assert len(res) == 3
|
assert len(res) == 3
|
||||||
@@ -645,7 +702,7 @@ def test_fts_on_list(mem_db: DBConnection):
|
|||||||
def test_fts_ngram(mem_db: DBConnection):
|
def test_fts_ngram(mem_db: DBConnection):
|
||||||
data = pa.table({"text": ["hello world", "lance database", "lance is cool"]})
|
data = pa.table({"text": ["hello world", "lance database", "lance is cool"]})
|
||||||
table = mem_db.create_table("test", data=data)
|
table = mem_db.create_table("test", data=data)
|
||||||
table.create_fts_index("text", base_tokenizer="ngram")
|
table.create_fts_index("text", use_tantivy=False, base_tokenizer="ngram")
|
||||||
|
|
||||||
results = table.search("lan", query_type="fts").limit(10).to_list()
|
results = table.search("lan", query_type="fts").limit(10).to_list()
|
||||||
assert len(results) == 2
|
assert len(results) == 2
|
||||||
@@ -664,6 +721,7 @@ def test_fts_ngram(mem_db: DBConnection):
|
|||||||
# test setting min_ngram_length and prefix_only
|
# test setting min_ngram_length and prefix_only
|
||||||
table.create_fts_index(
|
table.create_fts_index(
|
||||||
"text",
|
"text",
|
||||||
|
use_tantivy=False,
|
||||||
base_tokenizer="ngram",
|
base_tokenizer="ngram",
|
||||||
replace=True,
|
replace=True,
|
||||||
ngram_min_length=2,
|
ngram_min_length=2,
|
||||||
@@ -828,7 +886,7 @@ def test_fts_query_to_json():
|
|||||||
|
|
||||||
|
|
||||||
def test_fts_fast_search(table):
|
def test_fts_fast_search(table):
|
||||||
table.create_fts_index("text")
|
table.create_fts_index("text", use_tantivy=False)
|
||||||
|
|
||||||
# Insert some unindexed data
|
# Insert some unindexed data
|
||||||
table.add(
|
table.add(
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ def sync_table(tmpdir_factory) -> Table:
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
table = db.create_table("test", data)
|
table = db.create_table("test", data)
|
||||||
table.create_fts_index("text", with_position=False)
|
table.create_fts_index("text", with_position=False, use_tantivy=False)
|
||||||
return table
|
return table
|
||||||
|
|
||||||
|
|
||||||
@@ -192,7 +192,7 @@ def table_with_id(tmpdir_factory) -> Table:
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
table = db.create_table("test_with_id", data)
|
table = db.create_table("test_with_id", data)
|
||||||
table.create_fts_index("text", with_position=False)
|
table.create_fts_index("text", with_position=False, use_tantivy=False)
|
||||||
return table
|
return table
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1385,7 +1385,7 @@ def test_query_timeout(tmp_path):
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
table = db.create_table("test", data)
|
table = db.create_table("test", data)
|
||||||
table.create_fts_index("text")
|
table.create_fts_index("text", use_tantivy=False)
|
||||||
|
|
||||||
with pytest.raises(Exception, match="Query timeout"):
|
with pytest.raises(Exception, match="Query timeout"):
|
||||||
table.search().where("text = 'a'").to_list(timeout=timedelta(0))
|
table.search().where("text = 'a'").to_list(timeout=timedelta(0))
|
||||||
|
|||||||
@@ -26,8 +26,11 @@ from lancedb.rerankers import (
|
|||||||
)
|
)
|
||||||
from lancedb.table import LanceTable
|
from lancedb.table import LanceTable
|
||||||
|
|
||||||
|
# Tests rely on FTS index
|
||||||
|
pytest.importorskip("lancedb.fts")
|
||||||
|
|
||||||
def get_test_table(tmp_path):
|
|
||||||
|
def get_test_table(tmp_path, use_tantivy):
|
||||||
db = lancedb.connect(tmp_path)
|
db = lancedb.connect(tmp_path)
|
||||||
# Create a LanceDB table schema with a vector and a text column
|
# Create a LanceDB table schema with a vector and a text column
|
||||||
emb = EmbeddingFunctionRegistry.get_instance().get("test").create()
|
emb = EmbeddingFunctionRegistry.get_instance().get("test").create()
|
||||||
@@ -95,7 +98,7 @@ def get_test_table(tmp_path):
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Create a fts index
|
# Create a fts index
|
||||||
table.create_fts_index("text", replace=True)
|
table.create_fts_index("text", use_tantivy=use_tantivy, replace=True)
|
||||||
|
|
||||||
return table, MyTable
|
return table, MyTable
|
||||||
|
|
||||||
@@ -205,8 +208,8 @@ def _run_test_reranker(reranker, table, query, query_vector, schema):
|
|||||||
assert len(result) == 20 and result == result_arrow
|
assert len(result) == 20 and result == result_arrow
|
||||||
|
|
||||||
|
|
||||||
def _run_test_hybrid_reranker(reranker, tmp_path):
|
def _run_test_hybrid_reranker(reranker, tmp_path, use_tantivy):
|
||||||
table, schema = get_test_table(tmp_path)
|
table, schema = get_test_table(tmp_path, use_tantivy)
|
||||||
# The default reranker
|
# The default reranker
|
||||||
result1 = (
|
result1 = (
|
||||||
table.search(
|
table.search(
|
||||||
@@ -282,7 +285,8 @@ def _run_test_hybrid_reranker(reranker, tmp_path):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_linear_combination(tmp_path):
|
@pytest.mark.parametrize("use_tantivy", [True, False])
|
||||||
|
def test_linear_combination(tmp_path, use_tantivy):
|
||||||
reranker = LinearCombinationReranker()
|
reranker = LinearCombinationReranker()
|
||||||
|
|
||||||
vector_results = pa.Table.from_pydict(
|
vector_results = pa.Table.from_pydict(
|
||||||
@@ -309,20 +313,22 @@ def test_linear_combination(tmp_path):
|
|||||||
assert "_score" not in combined_results.column_names
|
assert "_score" not in combined_results.column_names
|
||||||
assert "_relevance_score" in combined_results.column_names
|
assert "_relevance_score" in combined_results.column_names
|
||||||
|
|
||||||
_run_test_hybrid_reranker(reranker, tmp_path)
|
_run_test_hybrid_reranker(reranker, tmp_path, use_tantivy)
|
||||||
|
|
||||||
|
|
||||||
def test_rrf_reranker(tmp_path):
|
@pytest.mark.parametrize("use_tantivy", [True, False])
|
||||||
|
def test_rrf_reranker(tmp_path, use_tantivy):
|
||||||
reranker = RRFReranker()
|
reranker = RRFReranker()
|
||||||
_run_test_hybrid_reranker(reranker, tmp_path)
|
_run_test_hybrid_reranker(reranker, tmp_path, use_tantivy)
|
||||||
|
|
||||||
|
|
||||||
def test_mrr_reranker(tmp_path):
|
@pytest.mark.parametrize("use_tantivy", [True, False])
|
||||||
|
def test_mrr_reranker(tmp_path, use_tantivy):
|
||||||
reranker = MRRReranker()
|
reranker = MRRReranker()
|
||||||
_run_test_hybrid_reranker(reranker, tmp_path)
|
_run_test_hybrid_reranker(reranker, tmp_path, use_tantivy)
|
||||||
|
|
||||||
# Test multi-vector part
|
# Test multi-vector part
|
||||||
table, schema = get_test_table(tmp_path)
|
table, schema = get_test_table(tmp_path, use_tantivy)
|
||||||
query = "single player experience"
|
query = "single player experience"
|
||||||
rs1 = table.search(query, vector_column_name="vector").limit(10).with_row_id(True)
|
rs1 = table.search(query, vector_column_name="vector").limit(10).with_row_id(True)
|
||||||
rs2 = (
|
rs2 = (
|
||||||
@@ -357,7 +363,7 @@ def test_rrf_reranker_distance():
|
|||||||
table = db.create_table("test", data)
|
table = db.create_table("test", data)
|
||||||
|
|
||||||
table.create_index(num_partitions=1, num_sub_vectors=2)
|
table.create_index(num_partitions=1, num_sub_vectors=2)
|
||||||
table.create_fts_index("text")
|
table.create_fts_index("text", use_tantivy=False)
|
||||||
|
|
||||||
reranker = RRFReranker(return_score="all")
|
reranker = RRFReranker(return_score="all")
|
||||||
|
|
||||||
@@ -416,31 +422,35 @@ def test_rrf_reranker_distance():
|
|||||||
@pytest.mark.skipif(
|
@pytest.mark.skipif(
|
||||||
os.environ.get("COHERE_API_KEY") is None, reason="COHERE_API_KEY not set"
|
os.environ.get("COHERE_API_KEY") is None, reason="COHERE_API_KEY not set"
|
||||||
)
|
)
|
||||||
def test_cohere_reranker(tmp_path):
|
@pytest.mark.parametrize("use_tantivy", [True, False])
|
||||||
|
def test_cohere_reranker(tmp_path, use_tantivy):
|
||||||
pytest.importorskip("cohere")
|
pytest.importorskip("cohere")
|
||||||
reranker = CohereReranker()
|
reranker = CohereReranker()
|
||||||
table, schema = get_test_table(tmp_path)
|
table, schema = get_test_table(tmp_path, use_tantivy)
|
||||||
_run_test_reranker(reranker, table, "single player experience", None, schema)
|
_run_test_reranker(reranker, table, "single player experience", None, schema)
|
||||||
|
|
||||||
|
|
||||||
def test_cross_encoder_reranker(tmp_path):
|
@pytest.mark.parametrize("use_tantivy", [True, False])
|
||||||
|
def test_cross_encoder_reranker(tmp_path, use_tantivy):
|
||||||
pytest.importorskip("sentence_transformers")
|
pytest.importorskip("sentence_transformers")
|
||||||
reranker = CrossEncoderReranker()
|
reranker = CrossEncoderReranker()
|
||||||
table, schema = get_test_table(tmp_path)
|
table, schema = get_test_table(tmp_path, use_tantivy)
|
||||||
_run_test_reranker(reranker, table, "single player experience", None, schema)
|
_run_test_reranker(reranker, table, "single player experience", None, schema)
|
||||||
|
|
||||||
|
|
||||||
def test_colbert_reranker(tmp_path):
|
@pytest.mark.parametrize("use_tantivy", [True, False])
|
||||||
|
def test_colbert_reranker(tmp_path, use_tantivy):
|
||||||
pytest.importorskip("rerankers")
|
pytest.importorskip("rerankers")
|
||||||
reranker = ColbertReranker()
|
reranker = ColbertReranker()
|
||||||
table, schema = get_test_table(tmp_path)
|
table, schema = get_test_table(tmp_path, use_tantivy)
|
||||||
_run_test_reranker(reranker, table, "single player experience", None, schema)
|
_run_test_reranker(reranker, table, "single player experience", None, schema)
|
||||||
|
|
||||||
|
|
||||||
def test_answerdotai_reranker(tmp_path):
|
@pytest.mark.parametrize("use_tantivy", [True, False])
|
||||||
|
def test_answerdotai_reranker(tmp_path, use_tantivy):
|
||||||
pytest.importorskip("rerankers")
|
pytest.importorskip("rerankers")
|
||||||
reranker = AnswerdotaiRerankers()
|
reranker = AnswerdotaiRerankers()
|
||||||
table, schema = get_test_table(tmp_path)
|
table, schema = get_test_table(tmp_path, use_tantivy)
|
||||||
_run_test_reranker(reranker, table, "single player experience", None, schema)
|
_run_test_reranker(reranker, table, "single player experience", None, schema)
|
||||||
|
|
||||||
|
|
||||||
@@ -449,9 +459,10 @@ def test_answerdotai_reranker(tmp_path):
|
|||||||
or os.environ.get("OPENAI_BASE_URL") is not None,
|
or os.environ.get("OPENAI_BASE_URL") is not None,
|
||||||
reason="OPENAI_API_KEY not set",
|
reason="OPENAI_API_KEY not set",
|
||||||
)
|
)
|
||||||
def test_openai_reranker(tmp_path):
|
@pytest.mark.parametrize("use_tantivy", [True, False])
|
||||||
|
def test_openai_reranker(tmp_path, use_tantivy):
|
||||||
pytest.importorskip("openai")
|
pytest.importorskip("openai")
|
||||||
table, schema = get_test_table(tmp_path)
|
table, schema = get_test_table(tmp_path, use_tantivy)
|
||||||
reranker = OpenaiReranker()
|
reranker = OpenaiReranker()
|
||||||
_run_test_reranker(reranker, table, "single player experience", None, schema)
|
_run_test_reranker(reranker, table, "single player experience", None, schema)
|
||||||
|
|
||||||
@@ -459,9 +470,10 @@ def test_openai_reranker(tmp_path):
|
|||||||
@pytest.mark.skipif(
|
@pytest.mark.skipif(
|
||||||
os.environ.get("JINA_API_KEY") is None, reason="JINA_API_KEY not set"
|
os.environ.get("JINA_API_KEY") is None, reason="JINA_API_KEY not set"
|
||||||
)
|
)
|
||||||
def test_jina_reranker(tmp_path):
|
@pytest.mark.parametrize("use_tantivy", [True, False])
|
||||||
|
def test_jina_reranker(tmp_path, use_tantivy):
|
||||||
pytest.importorskip("jina")
|
pytest.importorskip("jina")
|
||||||
table, schema = get_test_table(tmp_path)
|
table, schema = get_test_table(tmp_path, use_tantivy)
|
||||||
reranker = JinaReranker()
|
reranker = JinaReranker()
|
||||||
_run_test_reranker(reranker, table, "single player experience", None, schema)
|
_run_test_reranker(reranker, table, "single player experience", None, schema)
|
||||||
|
|
||||||
@@ -469,10 +481,11 @@ def test_jina_reranker(tmp_path):
|
|||||||
@pytest.mark.skipif(
|
@pytest.mark.skipif(
|
||||||
os.environ.get("VOYAGE_API_KEY") is None, reason="VOYAGE_API_KEY not set"
|
os.environ.get("VOYAGE_API_KEY") is None, reason="VOYAGE_API_KEY not set"
|
||||||
)
|
)
|
||||||
def test_voyageai_reranker(tmp_path):
|
@pytest.mark.parametrize("use_tantivy", [True, False])
|
||||||
|
def test_voyageai_reranker(tmp_path, use_tantivy):
|
||||||
pytest.importorskip("voyageai")
|
pytest.importorskip("voyageai")
|
||||||
reranker = VoyageAIReranker(model_name="rerank-2.5")
|
reranker = VoyageAIReranker(model_name="rerank-2.5")
|
||||||
table, schema = get_test_table(tmp_path)
|
table, schema = get_test_table(tmp_path, use_tantivy)
|
||||||
_run_test_reranker(reranker, table, "single player experience", None, schema)
|
_run_test_reranker(reranker, table, "single player experience", None, schema)
|
||||||
|
|
||||||
|
|
||||||
@@ -491,7 +504,7 @@ def test_empty_result_reranker():
|
|||||||
|
|
||||||
# Create empty table with schema
|
# Create empty table with schema
|
||||||
empty_table = db.create_table("empty_table", schema=schema, mode="overwrite")
|
empty_table = db.create_table("empty_table", schema=schema, mode="overwrite")
|
||||||
empty_table.create_fts_index("text", replace=True)
|
empty_table.create_fts_index("text", use_tantivy=False, replace=True)
|
||||||
for reranker in [
|
for reranker in [
|
||||||
CrossEncoderReranker(),
|
CrossEncoderReranker(),
|
||||||
# ColbertReranker(),
|
# ColbertReranker(),
|
||||||
@@ -590,10 +603,11 @@ def test_empty_hybrid_result_reranker():
|
|||||||
assert "_rowid" in result.column_names
|
assert "_rowid" in result.column_names
|
||||||
|
|
||||||
|
|
||||||
def test_cross_encoder_reranker_return_all(tmp_path):
|
@pytest.mark.parametrize("use_tantivy", [True, False])
|
||||||
|
def test_cross_encoder_reranker_return_all(tmp_path, use_tantivy):
|
||||||
pytest.importorskip("sentence_transformers")
|
pytest.importorskip("sentence_transformers")
|
||||||
reranker = CrossEncoderReranker(return_score="all")
|
reranker = CrossEncoderReranker(return_score="all")
|
||||||
table, schema = get_test_table(tmp_path)
|
table, schema = get_test_table(tmp_path, use_tantivy)
|
||||||
query = "single player experience"
|
query = "single player experience"
|
||||||
result = (
|
result = (
|
||||||
table.search(query, query_type="hybrid", vector_column_name="vector")
|
table.search(query, query_type="hybrid", vector_column_name="vector")
|
||||||
|
|||||||
@@ -242,8 +242,8 @@ def test_s3_dynamodb_sync(s3_bucket: str, commit_table: str, monkeypatch):
|
|||||||
|
|
||||||
# FTS indices should error since they are not supported yet.
|
# FTS indices should error since they are not supported yet.
|
||||||
with pytest.raises(
|
with pytest.raises(
|
||||||
ValueError,
|
NotImplementedError,
|
||||||
match="Tantivy-based FTS has been removed",
|
match="Full-text search is only supported on the local filesystem",
|
||||||
):
|
):
|
||||||
table.create_fts_index("x", use_tantivy=True)
|
table.create_fts_index("x", use_tantivy=True)
|
||||||
|
|
||||||
|
|||||||
@@ -1948,6 +1948,7 @@ def setup_hybrid_search_table(db: DBConnection, embedding_func):
|
|||||||
|
|
||||||
def test_hybrid_search(tmp_db: DBConnection):
|
def test_hybrid_search(tmp_db: DBConnection):
|
||||||
# This test uses an FTS index
|
# This test uses an FTS index
|
||||||
|
pytest.importorskip("lancedb.fts")
|
||||||
pytest.importorskip("lance")
|
pytest.importorskip("lance")
|
||||||
|
|
||||||
table, MyTable, emb = setup_hybrid_search_table(tmp_db, "test")
|
table, MyTable, emb = setup_hybrid_search_table(tmp_db, "test")
|
||||||
@@ -2018,6 +2019,7 @@ def test_hybrid_search(tmp_db: DBConnection):
|
|||||||
|
|
||||||
def test_hybrid_search_metric_type(tmp_db: DBConnection):
|
def test_hybrid_search_metric_type(tmp_db: DBConnection):
|
||||||
# This test uses an FTS index
|
# This test uses an FTS index
|
||||||
|
pytest.importorskip("lancedb.fts")
|
||||||
pytest.importorskip("lance")
|
pytest.importorskip("lance")
|
||||||
|
|
||||||
# Need to use nonnorm as the embedding function so l2 and dot results
|
# Need to use nonnorm as the embedding function so l2 and dot results
|
||||||
|
|||||||
Generated
+40
@@ -1996,6 +1996,7 @@ tests = [
|
|||||||
{ name = "pytest-mock" },
|
{ name = "pytest-mock" },
|
||||||
{ name = "pytz" },
|
{ name = "pytz" },
|
||||||
{ name = "requests" },
|
{ name = "requests" },
|
||||||
|
{ name = "tantivy" },
|
||||||
]
|
]
|
||||||
|
|
||||||
[package.metadata]
|
[package.metadata]
|
||||||
@@ -2049,6 +2050,7 @@ requires-dist = [
|
|||||||
{ name = "sentence-transformers", marker = "extra == 'embeddings'", specifier = ">=2.2.0" },
|
{ name = "sentence-transformers", marker = "extra == 'embeddings'", specifier = ">=2.2.0" },
|
||||||
{ name = "sentencepiece", marker = "extra == 'embeddings'", specifier = ">=0.1.99" },
|
{ name = "sentencepiece", marker = "extra == 'embeddings'", specifier = ">=0.1.99" },
|
||||||
{ name = "sentencepiece", marker = "extra == 'siglip'" },
|
{ name = "sentencepiece", marker = "extra == 'siglip'" },
|
||||||
|
{ name = "tantivy", marker = "extra == 'tests'", specifier = ">=0.20.0" },
|
||||||
{ name = "torch", marker = "extra == 'clip'" },
|
{ name = "torch", marker = "extra == 'clip'" },
|
||||||
{ name = "torch", marker = "extra == 'embeddings'", specifier = ">=2.0.0" },
|
{ name = "torch", marker = "extra == 'embeddings'", specifier = ">=2.0.0" },
|
||||||
{ name = "torch", marker = "extra == 'siglip'" },
|
{ name = "torch", marker = "extra == 'siglip'" },
|
||||||
@@ -4777,6 +4779,44 @@ wheels = [
|
|||||||
{ url = "https://files.pythonhosted.org/packages/40/44/4a5f08c96eb108af5cb50b41f76142f0afa346dfa99d5296fe7202a11854/tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f", size = 35252, upload-time = "2022-10-06T17:21:44.262Z" },
|
{ url = "https://files.pythonhosted.org/packages/40/44/4a5f08c96eb108af5cb50b41f76142f0afa346dfa99d5296fe7202a11854/tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f", size = 35252, upload-time = "2022-10-06T17:21:44.262Z" },
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tantivy"
|
||||||
|
version = "0.25.1"
|
||||||
|
source = { registry = "https://pypi.org/simple" }
|
||||||
|
sdist = { url = "https://files.pythonhosted.org/packages/1b/f9/0cd3955d155d3e3ef74b864769514dd191e5dacba9f0beb7af2d914942ce/tantivy-0.25.1.tar.gz", hash = "sha256:68a3314699a7d18fcf338b52bae8ce46a97dde1128a3e47e33fa4db7f71f265e", size = 75120, upload-time = "2025-12-02T11:57:12.997Z" }
|
||||||
|
wheels = [
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/80/f7/2276bed3bed983ce2970dc70e3571f372587fe4f5f2bac1d6d617df08fa3/tantivy-0.25.1-cp310-cp310-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:7aa587a3dc9470584cacf5e3640fee93d12ec5f10109669c1f47c4e90820b958", size = 7638510, upload-time = "2025-12-02T11:56:08.754Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/20/8c/078dc50570e243414356b05633f52fe544b85179281ffa9f1fe05d76bbd8/tantivy-0.25.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:56d77fe667595693d9fa5f0b4545776d84da9526bab0273b3fc6c7536dc0d8a2", size = 3932659, upload-time = "2025-12-02T11:56:10.621Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/bd/dc/281c48436a1e3178b58fe463af314434fe0f3a4ec0c7588a362900e0c69e/tantivy-0.25.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5ba8c347cd48595fcaeabb28a909ebce92cf9c5e5c84ab5ba1136a280a307b5c", size = 4197430, upload-time = "2025-12-02T11:56:12.65Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/7b/6c/61e6e0b0a350007d10a9b66a35703361d3345e14e7a7cc83494776b2a054/tantivy-0.25.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa7c4932e8fde1f09f2d46226060e827e197c2749abdc6129d73a752773adc38", size = 4184055, upload-time = "2025-12-02T11:56:14.647Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/5f/fd/0eb059b12f0b6f91623a54a46448a83b7f716d08f3bca68c095d697b85da/tantivy-0.25.1-cp310-cp310-win_amd64.whl", hash = "sha256:afcfc5dbb0bcd5d24531f4471737ae0896f33528426ab0b1dad3e427c19120f6", size = 3424134, upload-time = "2025-12-02T11:56:16.242Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/4e/7a/8a277f377e8a151fc0e71d4ffc1114aefb6e5e1c7dd609fed0955cf34ed8/tantivy-0.25.1-cp311-cp311-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:d363d7b4207d3a5aa7f0d212420df35bed18bdb6bae26a2a8bd57428388b7c29", size = 7637033, upload-time = "2025-12-02T11:56:18.104Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/71/31/8b4acdedfc9f9a2d04b1340d07eef5213d6f151d1e18da0cb423e5f090d2/tantivy-0.25.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:8f4389cf1d889a1df7c5a3195806b4b56c37cee10d8a26faaa0dea35a867b5ff", size = 3932180, upload-time = "2025-12-02T11:56:19.833Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/2f/dc/3e8499c21b4b9795e8f2fc54c68ce5b92905aaeadadaa56ecfa9180b11b1/tantivy-0.25.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99864c09fc54652c3c2486cdf13f86cdc8200f4b481569cb291e095ca5d496e5", size = 4197620, upload-time = "2025-12-02T11:56:21.496Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/f8/8e/f2ce62fffc811eb62bead92c7b23c2e218f817cbd54c4f3b802e03ba1438/tantivy-0.25.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05abf37ddbc5063c575548be0d62931629c086bff7a5a1b67cf5a8f5ebf4cd8c", size = 4183794, upload-time = "2025-12-02T11:56:23.215Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/de/64/24e2891b0ba3fd9853e10c296095a33b89bf3efd65e29da1ee5dae736040/tantivy-0.25.1-cp311-cp311-win_amd64.whl", hash = "sha256:f307ee8ad21597b0be23af83008fd66cfd5f958cdfa24ec0aaa08a38e86bbef4", size = 3424235, upload-time = "2025-12-02T11:56:25.172Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/41/e7/6849c713ed0996c7628324c60512c4882006f0a62145e56c624a93407f90/tantivy-0.25.1-cp312-cp312-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:90fd919e5f611809f746560ecf36eb9be824dec62e21ae17a27243759edb9aa1", size = 7621494, upload-time = "2025-12-02T11:56:27.069Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/c5/22/c3d8294600dc6e7fa350daef9ff337d3c06e132b81df727de9f7a50c692a/tantivy-0.25.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:4613c7cf6c23f3a97989819690a0f956d799354957de7a204abcc60083cebe02", size = 3925219, upload-time = "2025-12-02T11:56:29.403Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/41/fc/cbb1df71dd44c9110eff4eaaeda9d44f2d06182fe0452193be20ddfba93f/tantivy-0.25.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c477bd20b4df804d57dfc5033431bef27cde605695ae141b03abbf6ebc069129", size = 4198699, upload-time = "2025-12-02T11:56:31.359Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/47/4d/71abb78b774073c3ce12a4faa4351a9d910a71ffa3659526affba163873d/tantivy-0.25.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9b1a1ba1113c523c7ff7b10f282d6c4074006f7ef8d71e1d973d51bf7291ddb", size = 4183585, upload-time = "2025-12-02T11:56:33.317Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/be/16/3f00cd7ec458b92a0e977960af9ddfbeb762127d9acc68da9094a1fda556/tantivy-0.25.1-cp312-cp312-win_amd64.whl", hash = "sha256:9de0bafd3bd7ac9f8f82d53e17562e9db11a5af308fe5185c4bd86feaddbe4a6", size = 3424622, upload-time = "2025-12-02T11:56:34.788Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/3d/25/73cfbcf1a8ea49be6c42817431cac46b70a119fe64da903fcc2d92b5b511/tantivy-0.25.1-cp313-cp313-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:f51ff7196c6f31719202080ed8372d5e3d51e92c749c032fb8234f012e99744c", size = 7622530, upload-time = "2025-12-02T11:56:36.839Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/12/c8/c0d7591cdf4f7e7a9fc4da786d1ca8cd1aacffaa2be16ea6d401a8e4a566/tantivy-0.25.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:550e63321bfcacc003859f2fa29c1e8e56450807b3c9a501c1add27cfb9236d9", size = 3925637, upload-time = "2025-12-02T11:56:38.425Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/3a/09/bedfc223bffec7641b417dd7ab071134b2ef8f8550e9b1fb6014657ef52e/tantivy-0.25.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fde31cc8d6e122faf7902aeea32bc008a429a6e8904e34d3468126a3ec01b016", size = 4197322, upload-time = "2025-12-02T11:56:40.411Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/f5/f1/1fa5183500c8042200c9f2b840d34f5bbcfb434a1ee750e7132262d2a5c9/tantivy-0.25.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b11bd5a518b0be645320b47af8493f6a40c4f3234313e37adcf4534a564d27dd", size = 4183143, upload-time = "2025-12-02T11:56:42.048Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/d5/74/a4c4f4eb95888ccb784da3b017aa0625ab1ac411bf5d022a9a797d9a2334/tantivy-0.25.1-cp313-cp313-win_amd64.whl", hash = "sha256:cc7fe88853e06b3251ee4fa42b7a2038727f850c8765bcc8167cfc73585dd24e", size = 3423491, upload-time = "2025-12-02T11:56:43.858Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/8b/2f/581519492226f97d23bd0adc95dad991ebeaa73ea6abc8bff389a3096d9a/tantivy-0.25.1-cp313-cp313t-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:dae99e75b7eaa9bf5bd16ab106b416370f08c135aed0e117d62a3201cd1ffe36", size = 7610316, upload-time = "2025-12-02T11:56:45.927Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/91/40/5d7bc315ab9e6a22c5572656e8ada1c836cfa96dccf533377504fbc3c9d9/tantivy-0.25.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:506e9533c5ef4d3df43bad64ffecc0aa97c76e361ea610815dc3a20a9d6b30b3", size = 3919882, upload-time = "2025-12-02T11:56:48.469Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/02/b9/e0ef2f57a6a72444cb66c2ffbc310ab33ffaace275f1c4b0319d84ea3f18/tantivy-0.25.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5dbd4f8f264dacbcc9dee542832da2173fd53deaaea03f082d95214f8b5ed6bc", size = 4196031, upload-time = "2025-12-02T11:56:50.151Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/1e/02/bf3f8cacfd08642e14a73f7956a3fb95d58119132c98c121b9065a1f8615/tantivy-0.25.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:824c643ccb640dd9e35e00c5d5054ddf3323f56fe4219d57d428a9eeea13d22c", size = 4183437, upload-time = "2025-12-02T11:56:51.818Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/9c/83/afa90e570198e2d1139dd567bec3c9cf44d8c54f63a649f16d711ede02f5/tantivy-0.25.1-cp313-cp313t-win_amd64.whl", hash = "sha256:09c987b840afcebac817836ac08407eff17272d8aa60ce6e291f89c81830221d", size = 3419409, upload-time = "2025-12-02T11:56:53.451Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/ff/44/9f1d67aa5030f7eebc966c863d1316a510a971dd8bb45651df4acdfae9ed/tantivy-0.25.1-cp314-cp314-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:7f5d29ae85dd0f23df8d15b3e7b341d4f9eb5a446bbb9640df48ac1f6d9e0c6c", size = 7623723, upload-time = "2025-12-02T11:56:55.066Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/db/30/6e085bd3ed9d12da3c91c185854abd70f9dfd35fb36a75ea98428d42c30b/tantivy-0.25.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:f2d2938fb69a74fc1bb36edfaf7f0d1596fa1264db0f377bda2195c58bcb6245", size = 3926243, upload-time = "2025-12-02T11:56:57.058Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/32/f5/a00d65433430f51718e5cc6938df571765d7c4e03aedec5aef4ab567aa9b/tantivy-0.25.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f5ff124c4802558e627091e780b362ca944169736caba5a372eef39a79d0ae0", size = 4207186, upload-time = "2025-12-02T11:56:58.803Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/19/63/61bdb12fc95f2a7f77bd419a5149bfa9f28caa76cb569bf2b6b06e1d033e/tantivy-0.25.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:43b80ef62a340416139c93d19264e5f808da48e04f9305f1092b8ed22be0a5be", size = 4187312, upload-time = "2025-12-02T11:57:00.595Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/b7/de/e39c0b01d59019bf5c38face8b81defbc4a68cebf5e0c53bcb2cd715a449/tantivy-0.25.1-cp314-cp314-win_amd64.whl", hash = "sha256:286b654f40c70c1e6b64b9bc7031ed0bf5c440f5bffeaeeee21a0ee6cc39f0e2", size = 3436535, upload-time = "2025-12-02T11:57:02.267Z" },
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "threadpoolctl"
|
name = "threadpoolctl"
|
||||||
version = "3.6.0"
|
version = "3.6.0"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "lancedb"
|
name = "lancedb"
|
||||||
version = "0.28.0-beta.9"
|
version = "0.28.0-beta.7"
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
description = "LanceDB: A serverless, low-latency vector database for AI applications"
|
description = "LanceDB: A serverless, low-latency vector database for AI applications"
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
|
|||||||
@@ -26,7 +26,6 @@ use crate::connection::NamespaceClientPushdownOperation;
|
|||||||
use crate::database::ReadConsistency;
|
use crate::database::ReadConsistency;
|
||||||
use crate::error::{Error, Result};
|
use crate::error::{Error, Result};
|
||||||
use crate::table::NativeTable;
|
use crate::table::NativeTable;
|
||||||
use lance::dataset::WriteMode;
|
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
BaseTable, CloneTableRequest, CreateTableMode, CreateTableRequest as DbCreateTableRequest,
|
BaseTable, CloneTableRequest, CreateTableMode, CreateTableRequest as DbCreateTableRequest,
|
||||||
@@ -185,7 +184,6 @@ impl Database for LanceNamespaceDatabase {
|
|||||||
async fn create_table(&self, request: DbCreateTableRequest) -> Result<Arc<dyn BaseTable>> {
|
async fn create_table(&self, request: DbCreateTableRequest) -> Result<Arc<dyn BaseTable>> {
|
||||||
let mut table_id = request.namespace_path.clone();
|
let mut table_id = request.namespace_path.clone();
|
||||||
table_id.push(request.name.clone());
|
table_id.push(request.name.clone());
|
||||||
let mut existing_table = None;
|
|
||||||
|
|
||||||
match request.mode {
|
match request.mode {
|
||||||
CreateTableMode::Create => {}
|
CreateTableMode::Create => {}
|
||||||
@@ -194,7 +192,20 @@ impl Database for LanceNamespaceDatabase {
|
|||||||
id: Some(table_id.clone()),
|
id: Some(table_id.clone()),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
existing_table = self.namespace.describe_table(describe_request).await.ok();
|
let describe_result = self.namespace.describe_table(describe_request).await;
|
||||||
|
if describe_result.is_ok() {
|
||||||
|
// Drop the existing table - must succeed
|
||||||
|
let drop_request = DropTableRequest {
|
||||||
|
id: Some(table_id.clone()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
self.namespace
|
||||||
|
.drop_table(drop_request)
|
||||||
|
.await
|
||||||
|
.map_err(|e| Error::Runtime {
|
||||||
|
message: format!("Failed to drop existing table for overwrite: {}", e),
|
||||||
|
})?;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
CreateTableMode::ExistOk(_) => {
|
CreateTableMode::ExistOk(_) => {
|
||||||
let describe_request = DescribeTableRequest {
|
let describe_request = DescribeTableRequest {
|
||||||
@@ -229,82 +240,40 @@ impl Database for LanceNamespaceDatabase {
|
|||||||
};
|
};
|
||||||
|
|
||||||
let (location, initial_storage_options, managed_versioning) = {
|
let (location, initial_storage_options, managed_versioning) = {
|
||||||
if let Some(response) = existing_table {
|
let response = self
|
||||||
let loc = response.location.ok_or_else(|| Error::Runtime {
|
.namespace
|
||||||
message: "Table location is missing from describe_table response".to_string(),
|
.declare_table(declare_request)
|
||||||
})?;
|
.await
|
||||||
let opts = response
|
.map_err(|e| {
|
||||||
.storage_options
|
let err_str = e.to_string();
|
||||||
.or_else(|| Some(self.storage_options.clone()))
|
if matches!(request.mode, CreateTableMode::Create)
|
||||||
.filter(|o| !o.is_empty());
|
&& (err_str.contains("already exists")
|
||||||
(loc, opts, response.managed_versioning)
|
|| err_str.contains("TableAlreadyExists")
|
||||||
} else {
|
|| err_str.contains("table already exists"))
|
||||||
match self.namespace.declare_table(declare_request).await {
|
|
||||||
Ok(response) => {
|
|
||||||
let loc = response.location.ok_or_else(|| Error::Runtime {
|
|
||||||
message: "Table location is missing from declare_table response"
|
|
||||||
.to_string(),
|
|
||||||
})?;
|
|
||||||
let opts = response
|
|
||||||
.storage_options
|
|
||||||
.or_else(|| Some(self.storage_options.clone()))
|
|
||||||
.filter(|o: &HashMap<String, String>| !o.is_empty());
|
|
||||||
(loc, opts, response.managed_versioning)
|
|
||||||
}
|
|
||||||
Err(e)
|
|
||||||
if matches!(request.mode, CreateTableMode::Create) && {
|
|
||||||
let err_str = e.to_string();
|
|
||||||
err_str.contains("already exists")
|
|
||||||
|| err_str.contains("TableAlreadyExists")
|
|
||||||
|| err_str.contains("table already exists")
|
|
||||||
} =>
|
|
||||||
{
|
{
|
||||||
let response = self
|
Error::TableAlreadyExists {
|
||||||
.namespace
|
name: request.name.clone(),
|
||||||
.describe_table(DescribeTableRequest {
|
|
||||||
id: Some(table_id.clone()),
|
|
||||||
..Default::default()
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.map_err(|describe_err| Error::Runtime {
|
|
||||||
message: format!(
|
|
||||||
"Failed to describe existing declared table after declare conflict: {}",
|
|
||||||
describe_err
|
|
||||||
),
|
|
||||||
})?;
|
|
||||||
|
|
||||||
if response.version.is_some() && response.schema.is_some() {
|
|
||||||
return Err(Error::TableAlreadyExists {
|
|
||||||
name: request.name.clone(),
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
let loc = response.location.ok_or_else(|| Error::Runtime {
|
Error::Runtime {
|
||||||
message: "Table location is missing from describe_table response"
|
|
||||||
.to_string(),
|
|
||||||
})?;
|
|
||||||
let opts = response
|
|
||||||
.storage_options
|
|
||||||
.or_else(|| Some(self.storage_options.clone()))
|
|
||||||
.filter(|o: &HashMap<String, String>| !o.is_empty());
|
|
||||||
(loc, opts, response.managed_versioning)
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
return Err(Error::Runtime {
|
|
||||||
message: format!("Failed to declare table: {}", e),
|
message: format!("Failed to declare table: {}", e),
|
||||||
});
|
}
|
||||||
}
|
}
|
||||||
}
|
})?;
|
||||||
}
|
let loc = response.location.ok_or_else(|| Error::Runtime {
|
||||||
|
message: "Table location is missing from declare_table response".to_string(),
|
||||||
|
})?;
|
||||||
|
// Use storage options from response, fall back to self.storage_options
|
||||||
|
let opts = response
|
||||||
|
.storage_options
|
||||||
|
.or_else(|| Some(self.storage_options.clone()))
|
||||||
|
.filter(|o| !o.is_empty());
|
||||||
|
(loc, opts, response.managed_versioning)
|
||||||
};
|
};
|
||||||
|
|
||||||
// Build write params with storage options and commit handler
|
// Build write params with storage options and commit handler
|
||||||
let mut params = request.write_options.lance_write_params.unwrap_or_default();
|
let mut params = request.write_options.lance_write_params.unwrap_or_default();
|
||||||
|
|
||||||
if matches!(request.mode, CreateTableMode::Overwrite) {
|
|
||||||
params.mode = WriteMode::Overwrite;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Set up storage options if provided
|
// Set up storage options if provided
|
||||||
if let Some(storage_opts) = initial_storage_options {
|
if let Some(storage_opts) = initial_storage_options {
|
||||||
let store_params = params
|
let store_params = params
|
||||||
@@ -761,58 +730,6 @@ mod tests {
|
|||||||
assert_eq!(id_col.value(2), 30);
|
assert_eq!(id_col.value(2), 30);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn test_namespace_create_table_after_declare_conflict() {
|
|
||||||
let tmp_dir = tempdir().unwrap();
|
|
||||||
let root_path = tmp_dir.path().to_str().unwrap().to_string();
|
|
||||||
|
|
||||||
let mut properties = HashMap::new();
|
|
||||||
properties.insert("root".to_string(), root_path);
|
|
||||||
|
|
||||||
let conn = connect_namespace("dir", properties)
|
|
||||||
.execute()
|
|
||||||
.await
|
|
||||||
.expect("Failed to connect to namespace");
|
|
||||||
|
|
||||||
conn.create_namespace(CreateNamespaceRequest {
|
|
||||||
id: Some(vec!["test_ns".into()]),
|
|
||||||
..Default::default()
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.expect("Failed to create namespace");
|
|
||||||
|
|
||||||
let namespace_client = conn.namespace_client().await.unwrap();
|
|
||||||
namespace_client
|
|
||||||
.declare_table(DeclareTableRequest {
|
|
||||||
id: Some(vec!["test_ns".into(), "declared_test".into()]),
|
|
||||||
..Default::default()
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.expect("Failed to declare table");
|
|
||||||
|
|
||||||
let test_data = create_test_data();
|
|
||||||
let table = conn
|
|
||||||
.create_table("declared_test", test_data)
|
|
||||||
.namespace(vec!["test_ns".into()])
|
|
||||||
.execute()
|
|
||||||
.await
|
|
||||||
.expect("Failed to create table after declare conflict");
|
|
||||||
|
|
||||||
let results = table
|
|
||||||
.query()
|
|
||||||
.execute()
|
|
||||||
.await
|
|
||||||
.expect("Failed to query table")
|
|
||||||
.try_collect::<Vec<_>>()
|
|
||||||
.await
|
|
||||||
.expect("Failed to collect results");
|
|
||||||
|
|
||||||
assert_eq!(results.len(), 1);
|
|
||||||
assert_eq!(results[0].num_rows(), 5);
|
|
||||||
assert_eq!(table.namespace(), &["test_ns"]);
|
|
||||||
assert_eq!(table.id(), "test_ns$declared_test");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_namespace_create_table_exist_ok_mode() {
|
async fn test_namespace_create_table_exist_ok_mode() {
|
||||||
// Setup: Create a temporary directory for the namespace
|
// Setup: Create a temporary directory for the namespace
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ use crate::remote::retry::{ResolvedRetryConfig, RetryCounter};
|
|||||||
const REQUEST_ID_HEADER: HeaderName = HeaderName::from_static("x-request-id");
|
const REQUEST_ID_HEADER: HeaderName = HeaderName::from_static("x-request-id");
|
||||||
|
|
||||||
/// Configuration for TLS/mTLS settings.
|
/// Configuration for TLS/mTLS settings.
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug, Default)]
|
||||||
pub struct TlsConfig {
|
pub struct TlsConfig {
|
||||||
/// Path to the client certificate file (PEM format)
|
/// Path to the client certificate file (PEM format)
|
||||||
pub cert_file: Option<String>,
|
pub cert_file: Option<String>,
|
||||||
@@ -24,22 +24,10 @@ pub struct TlsConfig {
|
|||||||
pub key_file: Option<String>,
|
pub key_file: Option<String>,
|
||||||
/// Path to the CA certificate file for server verification (PEM format)
|
/// Path to the CA certificate file for server verification (PEM format)
|
||||||
pub ssl_ca_cert: Option<String>,
|
pub ssl_ca_cert: Option<String>,
|
||||||
/// Whether to verify the hostname in the server's certificate.
|
/// Whether to verify the hostname in the server's certificate
|
||||||
/// Defaults to `true`.
|
|
||||||
pub assert_hostname: bool,
|
pub assert_hostname: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for TlsConfig {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self {
|
|
||||||
cert_file: None,
|
|
||||||
key_file: None,
|
|
||||||
ssl_ca_cert: None,
|
|
||||||
assert_hostname: true,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Trait for providing custom headers for each request
|
/// Trait for providing custom headers for each request
|
||||||
#[async_trait::async_trait]
|
#[async_trait::async_trait]
|
||||||
pub trait HeaderProvider: Send + Sync + std::fmt::Debug {
|
pub trait HeaderProvider: Send + Sync + std::fmt::Debug {
|
||||||
@@ -938,7 +926,7 @@ mod tests {
|
|||||||
assert!(config.cert_file.is_none());
|
assert!(config.cert_file.is_none());
|
||||||
assert!(config.key_file.is_none());
|
assert!(config.key_file.is_none());
|
||||||
assert!(config.ssl_ca_cert.is_none());
|
assert!(config.ssl_ca_cert.is_none());
|
||||||
assert!(config.assert_hostname);
|
assert!(!config.assert_hostname);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
Reference in New Issue
Block a user