mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-20 18:52:28 +00:00
Merge main into gatekeeper/fix-4260-1
Resolve Lance pins to beta.21 and refresh the vendored Arrow crate while preserving the sliced-child validity fix and regression tests.
This commit is contained in:
Gatefixer
+1
-1
@@ -1,5 +1,5 @@
|
||||
[tool.bumpversion]
|
||||
current_version = "0.41.0-beta.7"
|
||||
current_version = "0.41.0-beta.9"
|
||||
parse = """(?x)
|
||||
(?P<major>0|[1-9]\\d*)\\.
|
||||
(?P<minor>0|[1-9]\\d*)\\.
|
||||
|
||||
@@ -5,11 +5,15 @@ on:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- ci/check_cargo_lock.py
|
||||
- ci/check_sdist.py
|
||||
- ci/set_lance_version.py
|
||||
- ci/tests/**
|
||||
- .github/workflows/ci-scripts.yml
|
||||
pull_request:
|
||||
paths:
|
||||
- ci/check_cargo_lock.py
|
||||
- ci/check_sdist.py
|
||||
- ci/set_lance_version.py
|
||||
- ci/tests/**
|
||||
- .github/workflows/ci-scripts.yml
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
name: Docs prose style
|
||||
|
||||
# Vale checks the mechanical rules in the LanceDB style guide
|
||||
# (lancedb/docs, STYLE.md) with the package pinned in .vale.ini. Its findings
|
||||
# are advisory annotations on the lines a pull request adds or changes and
|
||||
# never fail the job; an install, sync or configuration error still does.
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- "docs/web/**/*.mdx"
|
||||
- ".vale.ini"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
env:
|
||||
VALE_VERSION: 3.24.0
|
||||
VALE_SHA256: 867534ddb678abca7f214bc4706ead0922bd5252ffe8035fc227013f66c8b214
|
||||
|
||||
jobs:
|
||||
prose:
|
||||
name: Lint changed prose
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install Vale
|
||||
run: |
|
||||
archive="vale_${VALE_VERSION}_Linux_64-bit.tar.gz"
|
||||
curl --fail --location --silent --show-error --output "$RUNNER_TEMP/$archive" \
|
||||
"https://github.com/vale-cli/vale/releases/download/v${VALE_VERSION}/$archive"
|
||||
echo "${VALE_SHA256} $RUNNER_TEMP/$archive" | sha256sum --check
|
||||
tar -xzf "$RUNNER_TEMP/$archive" -C "$RUNNER_TEMP" vale
|
||||
echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Annotate changed lines
|
||||
uses: vale-cli/vale-action@518a9136acc6e6668ce7c00d367051e0941e87ff # v3.0.0
|
||||
with:
|
||||
version: none
|
||||
files: docs/web
|
||||
reporter: github-pr-annotations
|
||||
filter_mode: added
|
||||
fail_on_error: false
|
||||
@@ -12,6 +12,9 @@ on:
|
||||
- .github/workflows/build_mac_wheel/action.yml
|
||||
- .github/workflows/build_windows_wheel/action.yml
|
||||
- .github/actions/setup-protoc/action.yml
|
||||
- ci/check_cargo_lock.py
|
||||
- ci/check_sdist.py
|
||||
- python/pyproject.toml
|
||||
- Cargo.toml # Change in dependency frequently breaks builds
|
||||
- Cargo.lock
|
||||
|
||||
@@ -161,10 +164,91 @@ jobs:
|
||||
name: wheels-windows
|
||||
path: target/wheels/lancedb-*.whl
|
||||
if-no-files-found: error
|
||||
sdist:
|
||||
name: Python sdist
|
||||
timeout-minutes: 60
|
||||
runs-on: ubuntu-22.04
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
lfs: true
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- name: Build sdist
|
||||
uses: PyO3/maturin-action@v1
|
||||
with:
|
||||
maturin-version: "1.12.4"
|
||||
command: sdist
|
||||
working-directory: python
|
||||
args: --out dist
|
||||
# maturin ships the nodejs-only pins in Cargo.lock. Remove this step when
|
||||
# the pinned maturin fixes https://github.com/PyO3/maturin/issues/2609.
|
||||
- name: Sync Cargo.lock to the packaged workspace
|
||||
shell: bash
|
||||
run: |
|
||||
# A repo lock in sync with every manifest means the prune only drops pins.
|
||||
cargo metadata --locked --format-version 1 > /dev/null
|
||||
mkdir "$RUNNER_TEMP/sdist-src"
|
||||
tar -xzf python/dist/lancedb-*.tar.gz -C "$RUNNER_TEMP/sdist-src"
|
||||
lock=$(echo "$RUNNER_TEMP"/sdist-src/lancedb-*/Cargo.lock)
|
||||
cp "$lock" "$RUNNER_TEMP/Cargo.lock.before"
|
||||
cargo metadata --format-version 1 \
|
||||
--manifest-path "$RUNNER_TEMP"/sdist-src/lancedb-*/Cargo.toml > /dev/null
|
||||
python ci/check_cargo_lock.py "$RUNNER_TEMP/Cargo.lock.before" "$lock"
|
||||
tar -czf python/dist/lancedb-*.tar.gz -C "$RUNNER_TEMP/sdist-src" \
|
||||
"$(basename python/dist/lancedb-*.tar.gz .tar.gz)"
|
||||
- name: Install sdist check tools
|
||||
shell: bash
|
||||
run: python -m pip install twine==7.0.0 pydistcheck==0.11.3
|
||||
- name: Check sdist size
|
||||
shell: bash
|
||||
run: |
|
||||
pydistcheck --select distro-too-large-compressed \
|
||||
--max-allowed-size-compressed 100M python/dist/lancedb-*.tar.gz
|
||||
- name: Check sdist contents
|
||||
shell: bash
|
||||
run: python ci/check_sdist.py python/dist/lancedb-*.tar.gz
|
||||
- name: Check sdist with twine
|
||||
shell: bash
|
||||
run: python -m twine check --strict python/dist/lancedb-*.tar.gz
|
||||
- uses: ./.github/actions/setup-protoc
|
||||
# Build the way `pip install lancedb` does when no wheel matches. The dev
|
||||
# profile keeps the build within the runner's disk.
|
||||
- name: Build wheel from sdist
|
||||
shell: bash
|
||||
env:
|
||||
MATURIN_PEP517_ARGS: "--profile dev --locked"
|
||||
CARGO_PROFILE_DEV_DEBUG: "0"
|
||||
run: |
|
||||
python -m pip wheel --no-deps --wheel-dir "$RUNNER_TEMP/sdist-wheel" \
|
||||
python/dist/lancedb-*.tar.gz
|
||||
- name: Smoke test wheel built from sdist
|
||||
shell: bash
|
||||
working-directory: ${{ runner.temp }}
|
||||
run: |
|
||||
python -m venv sdist-venv
|
||||
sdist-venv/bin/pip install sdist-wheel/lancedb-*.whl
|
||||
sdist-venv/bin/python -c '
|
||||
import tempfile
|
||||
import lancedb
|
||||
db = lancedb.connect(tempfile.mkdtemp())
|
||||
table = db.create_table("t", [{"vector": [1.0, 2.0], "id": 1}, {"vector": [3.0, 4.0], "id": 2}])
|
||||
assert table.search([1.0, 2.0]).limit(1).to_list()[0]["id"] == 1
|
||||
print("lancedb", lancedb.__version__, "built from sdist OK")
|
||||
'
|
||||
- uses: actions/upload-artifact@v7
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
with:
|
||||
name: sdist
|
||||
path: python/dist/lancedb-*.tar.gz
|
||||
if-no-files-found: error
|
||||
publish:
|
||||
name: Publish wheels
|
||||
name: Publish wheels and sdist
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
needs: [linux, mac, windows]
|
||||
needs: [linux, mac, windows, sdist]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
id-token: write
|
||||
@@ -177,7 +261,12 @@ jobs:
|
||||
pattern: wheels-*
|
||||
path: target/wheels
|
||||
merge-multiple: true
|
||||
- name: List wheels
|
||||
- name: Download sdist artifact
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: sdist
|
||||
path: target/wheels
|
||||
- name: List packages
|
||||
run: ls -la target/wheels
|
||||
- name: Choose repo
|
||||
id: choose_repo
|
||||
@@ -194,13 +283,18 @@ jobs:
|
||||
run: |
|
||||
shopt -s nullglob
|
||||
WHEELS=(target/wheels/*.whl)
|
||||
SDISTS=(target/wheels/*.tar.gz)
|
||||
if [[ ${#WHEELS[@]} -eq 0 ]]; then
|
||||
echo "No wheels found in target/wheels/" >&2
|
||||
exit 1
|
||||
fi
|
||||
for WHEEL in "${WHEELS[@]}"; do
|
||||
echo "Uploading $WHEEL to Fury"
|
||||
curl -f -F package=@"$WHEEL" "https://$FURY_TOKEN@push.fury.io/lancedb/"
|
||||
if [[ ${#SDISTS[@]} -ne 1 ]]; then
|
||||
echo "Expected one sdist in target/wheels/, found ${#SDISTS[@]}" >&2
|
||||
exit 1
|
||||
fi
|
||||
for PACKAGE in "${WHEELS[@]}" "${SDISTS[@]}"; do
|
||||
echo "Uploading $PACKAGE to Fury"
|
||||
curl -f -F package=@"$PACKAGE" "https://$FURY_TOKEN@push.fury.io/lancedb/"
|
||||
done
|
||||
# NOTE: pypa/gh-action-pypi-publish must be invoked directly from a
|
||||
# workflow file, not from inside a composite action. When called from a
|
||||
@@ -215,7 +309,7 @@ jobs:
|
||||
report-failure:
|
||||
name: Report Workflow Failure
|
||||
runs-on: ubuntu-latest
|
||||
needs: [linux, mac, windows, publish]
|
||||
needs: [linux, mac, windows, sdist, publish]
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
|
||||
@@ -40,3 +40,6 @@ dist
|
||||
target
|
||||
|
||||
**/sccache.log
|
||||
|
||||
# Vale styles installed by `vale sync`
|
||||
/.vale/
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
# Prose style for the pages in docs/web: the shared LanceDB package, pinned by
|
||||
# release. The rules and guide live in lancedb/docs (STYLE.md and vale/).
|
||||
# Run `vale sync` once, then `vale docs/web/<page>.mdx`.
|
||||
StylesPath = .vale/styles
|
||||
Packages = https://github.com/lancedb/docs/releases/download/vale-lancedb-v0.1.0/LanceDB.zip
|
||||
@@ -88,6 +88,15 @@ npm i -g mint # https://mintlify.com/docs/installation
|
||||
cd docs/web && mint dev
|
||||
```
|
||||
|
||||
Wording, names and punctuation follow the
|
||||
[LanceDB style guide](https://github.com/lancedb/docs/blob/main/STYLE.md).
|
||||
[Vale](https://vale.sh) checks part of it, and pull requests show its
|
||||
suggestions on the lines they change. To check a page yourself:
|
||||
|
||||
```bash
|
||||
vale sync && vale docs/web/indexing/vector-index.mdx
|
||||
```
|
||||
|
||||
This repository holds a page at every path the site publishes, so the whole
|
||||
navigation resolves locally. Enterprise pages exist here as open-source pages --
|
||||
what the capability is, and what the embedded form does instead. A separate
|
||||
|
||||
Generated
+49
-48
@@ -3584,8 +3584,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
|
||||
[[package]]
|
||||
name = "fsst"
|
||||
version = "14.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d"
|
||||
version = "14.0.0-beta.21"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"rand 0.9.5",
|
||||
@@ -5059,8 +5059,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
|
||||
|
||||
[[package]]
|
||||
name = "lance"
|
||||
version = "14.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d"
|
||||
version = "14.0.0-beta.21"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -5092,6 +5092,7 @@ dependencies = [
|
||||
"either",
|
||||
"fst",
|
||||
"futures",
|
||||
"hex",
|
||||
"humantime",
|
||||
"itertools 0.14.0",
|
||||
"lance-arrow",
|
||||
@@ -5133,7 +5134,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-arrow"
|
||||
version = "14.0.0-beta.19"
|
||||
version = "14.0.0-beta.21"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5155,7 +5156,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-scalar"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5169,7 +5170,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-stats"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5178,8 +5179,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-bitpacking"
|
||||
version = "14.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d"
|
||||
version = "14.0.0-beta.21"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
|
||||
dependencies = [
|
||||
"arrayref",
|
||||
"crunchy",
|
||||
@@ -5189,8 +5190,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-core"
|
||||
version = "14.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d"
|
||||
version = "14.0.0-beta.21"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5227,8 +5228,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datafusion"
|
||||
version = "14.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d"
|
||||
version = "14.0.0-beta.21"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5259,8 +5260,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datagen"
|
||||
version = "14.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d"
|
||||
version = "14.0.0-beta.21"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5277,8 +5278,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-derive"
|
||||
version = "14.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d"
|
||||
version = "14.0.0-beta.21"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -5287,8 +5288,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-encoding"
|
||||
version = "14.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d"
|
||||
version = "14.0.0-beta.21"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5321,8 +5322,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-file"
|
||||
version = "14.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d"
|
||||
version = "14.0.0-beta.21"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5354,8 +5355,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-geo"
|
||||
version = "14.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d"
|
||||
version = "14.0.0-beta.21"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
|
||||
dependencies = [
|
||||
"datafusion",
|
||||
"geo-traits",
|
||||
@@ -5369,8 +5370,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index"
|
||||
version = "14.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d"
|
||||
version = "14.0.0-beta.21"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -5441,8 +5442,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index-core"
|
||||
version = "14.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d"
|
||||
version = "14.0.0-beta.21"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5465,8 +5466,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-io"
|
||||
version = "14.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d"
|
||||
version = "14.0.0-beta.21"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5509,8 +5510,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-jsonb"
|
||||
version = "14.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d"
|
||||
version = "14.0.0-beta.21"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"ethnum",
|
||||
@@ -5527,8 +5528,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-linalg"
|
||||
version = "14.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d"
|
||||
version = "14.0.0-beta.21"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5542,8 +5543,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace"
|
||||
version = "14.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d"
|
||||
version = "14.0.0-beta.21"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -5557,8 +5558,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace-impls"
|
||||
version = "14.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d"
|
||||
version = "14.0.0-beta.21"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -5612,8 +5613,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-select"
|
||||
version = "14.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d"
|
||||
version = "14.0.0-beta.21"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5627,8 +5628,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-table"
|
||||
version = "14.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d"
|
||||
version = "14.0.0-beta.21"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5668,8 +5669,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-testing"
|
||||
version = "14.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d"
|
||||
version = "14.0.0-beta.21"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5682,8 +5683,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-tokenizer"
|
||||
version = "14.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d"
|
||||
version = "14.0.0-beta.21"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
|
||||
dependencies = [
|
||||
"frostem",
|
||||
"icu_segmenter",
|
||||
@@ -5696,7 +5697,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lancedb"
|
||||
version = "0.41.0-beta.7"
|
||||
version = "0.41.0-beta.9"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"anyhow",
|
||||
@@ -5794,7 +5795,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lancedb-nodejs"
|
||||
version = "0.41.0-beta.7"
|
||||
version = "0.41.0-beta.9"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5819,7 +5820,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lancedb-python"
|
||||
version = "0.41.0-beta.7"
|
||||
version = "0.41.0-beta.9"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
|
||||
+15
-15
@@ -13,21 +13,21 @@ categories = ["database-implementations"]
|
||||
rust-version = "1.94.0"
|
||||
|
||||
[workspace.dependencies]
|
||||
lance = { "version" = "=14.0.0-beta.19", default-features = false, "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=14.0.0-beta.19", default-features = false, "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-jsonb = { version = "=14.0.0-beta.19", tag = "v14.0.0-beta.19", git = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=14.0.0-beta.19", default-features = false, "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { version = "=14.0.0-beta.19", path = "python/vendor/lance-arrow" }
|
||||
lance = { "version" = "=14.0.0-beta.21", default-features = false, "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=14.0.0-beta.21", default-features = false, "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-jsonb = { version = "=14.0.0-beta.21", tag = "v14.0.0-beta.21", git = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=14.0.0-beta.21", default-features = false, "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { version = "=14.0.0-beta.21", path = "python/vendor/lance-arrow" }
|
||||
lancedb = { path = "rust/lancedb", default-features = false }
|
||||
ahash = "0.8"
|
||||
# Note that this one does not include pyarrow
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Fail if pruning Cargo.lock did more than remove packages and the edges to them.
|
||||
|
||||
Usage: python ci/check_cargo_lock.py <lock-before> <lock-after>
|
||||
"""
|
||||
|
||||
import sys
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def key(package: dict) -> tuple:
|
||||
return tuple(package.get(k) for k in ("name", "version", "source", "checksum"))
|
||||
|
||||
|
||||
def graph(lock: dict) -> dict[tuple, set[tuple]]:
|
||||
"""Map each package to the packages its dependency strings resolve to."""
|
||||
by_name = {}
|
||||
for p in lock["package"]:
|
||||
by_name.setdefault(p["name"], []).append(p)
|
||||
|
||||
def resolve(dep: str) -> tuple:
|
||||
# Cargo writes "name", "name version" or "name version (source)".
|
||||
name, *rest = dep.split(" ", 2)
|
||||
(match,) = [
|
||||
p
|
||||
for p in by_name[name]
|
||||
if (not rest or p["version"] == rest[0])
|
||||
and (len(rest) < 2 or rest[1] == f"({p.get('source')})")
|
||||
]
|
||||
return key(match)
|
||||
|
||||
return {
|
||||
key(p): {resolve(d) for d in p.get("dependencies", [])} for p in lock["package"]
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
before, after = (tomllib.loads(Path(p).read_text()) for p in sys.argv[1:])
|
||||
errors = []
|
||||
for field in (before.keys() | after.keys()) - {"package"}:
|
||||
if before.get(field) != after.get(field):
|
||||
errors.append(f"top-level {field!r} changed")
|
||||
edges_before, edges_after = graph(before), graph(after)
|
||||
removed = edges_before.keys() - edges_after.keys()
|
||||
for package, deps in edges_after.items():
|
||||
if package not in edges_before:
|
||||
errors.append(f"added or changed {package}")
|
||||
elif deps != edges_before[package] - removed:
|
||||
errors.append(f"dependencies of {package[:2]} changed")
|
||||
if errors:
|
||||
sys.exit("error: the prune changed Cargo.lock:\n" + "\n".join(errors))
|
||||
print(f"Cargo.lock prune only removed {len(removed)} packages")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Fail if the lancedb sdist lacks a git-tracked file of the Python package.
|
||||
|
||||
Usage: python ci/check_sdist.py <sdist> (run from the repository root)
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
from pathlib import Path
|
||||
|
||||
# Where git tracks the package, and where maturin puts it under the sdist root.
|
||||
TRACKED_PACKAGE = "python/python/lancedb"
|
||||
SDIST_PACKAGE = "python/lancedb"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
sdist = Path(sys.argv[1])
|
||||
root = sdist.name.removesuffix(".tar.gz")
|
||||
listing = subprocess.run(
|
||||
["git", "ls-files", "-z", "--", TRACKED_PACKAGE],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout
|
||||
tracked = [p for p in listing.split("\0") if p]
|
||||
if not tracked:
|
||||
sys.exit(f"error: git tracks no files under {TRACKED_PACKAGE}")
|
||||
|
||||
with tarfile.open(sdist) as tar:
|
||||
members = {m.name for m in tar if m.isfile()}
|
||||
missing = [
|
||||
p
|
||||
for p in tracked
|
||||
if f"{root}/{SDIST_PACKAGE}{p.removeprefix(TRACKED_PACKAGE)}" not in members
|
||||
]
|
||||
if missing:
|
||||
sys.exit(f"error: {sdist.name} lacks tracked sources: {', '.join(missing)}")
|
||||
print(f"{sdist.name} OK: all {len(tracked)} tracked sources present")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,97 @@
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPT = REPO_ROOT / "ci" / "check_cargo_lock.py"
|
||||
CRATES_IO = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
|
||||
def package(name, version, deps=(), source=CRATES_IO, checksum="aa"):
|
||||
text = f'[[package]]\nname = "{name}"\nversion = "{version}"\n'
|
||||
if source:
|
||||
text += f'source = "{source}"\nchecksum = "{checksum}"\n'
|
||||
if deps:
|
||||
text += f"dependencies = {list(deps)!r}\n".replace("'", '"')
|
||||
return text
|
||||
|
||||
|
||||
LIBC = package("libc", "0.2.1")
|
||||
UNTRUSTED = package("untrusted", "0.9.0")
|
||||
# The prune drops napi and untrusted 0.7.1, the edge from app to it, and the
|
||||
# version qualifier on app's remaining untrusted edge.
|
||||
BEFORE = "version = 4\n\n" + "".join(
|
||||
[
|
||||
package(
|
||||
"app", "1.0.0", ["libc", "untrusted 0.7.1", "untrusted 0.9.0"], source=None
|
||||
),
|
||||
package("napi", "3.0.0", ["untrusted 0.7.1"]),
|
||||
LIBC,
|
||||
package("untrusted", "0.7.1"),
|
||||
UNTRUSTED,
|
||||
]
|
||||
)
|
||||
APP = package("app", "1.0.0", ["libc", "untrusted"], source=None)
|
||||
|
||||
|
||||
class CheckCargoLockTest(unittest.TestCase):
|
||||
def run_check(self, after, header="version = 4\n\n"):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
before_path = Path(tmp, "before.lock")
|
||||
after_path = Path(tmp, "after.lock")
|
||||
before_path.write_text(BEFORE)
|
||||
after_path.write_text(header + after)
|
||||
return subprocess.run(
|
||||
[sys.executable, SCRIPT, before_path, after_path],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
def test_prune_passes(self):
|
||||
result = self.run_check(APP + LIBC + UNTRUSTED)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
|
||||
def test_added_entry_fails(self):
|
||||
result = self.run_check(APP + LIBC + UNTRUSTED + package("ring", "0.17.0"))
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("ring", result.stderr)
|
||||
|
||||
def test_changed_version_fails(self):
|
||||
result = self.run_check(APP + package("libc", "0.2.2") + UNTRUSTED)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("0.2.2", result.stderr)
|
||||
|
||||
def test_changed_source_fails(self):
|
||||
libc = package("libc", "0.2.1", source="git+https://example.com/libc")
|
||||
result = self.run_check(APP + libc + UNTRUSTED)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("example.com", result.stderr)
|
||||
|
||||
def test_changed_checksum_fails(self):
|
||||
libc = package("libc", "0.2.1", checksum="bb")
|
||||
result = self.run_check(APP + libc + UNTRUSTED)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("'bb'", result.stderr)
|
||||
|
||||
def test_added_dependency_fails(self):
|
||||
libc = package("libc", "0.2.1", ["untrusted"])
|
||||
result = self.run_check(APP + libc + UNTRUSTED)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("dependencies of ('libc'", result.stderr)
|
||||
|
||||
def test_removed_dependency_fails(self):
|
||||
app = package("app", "1.0.0", ["untrusted"], source=None)
|
||||
result = self.run_check(app + LIBC + UNTRUSTED)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("dependencies of ('app'", result.stderr)
|
||||
|
||||
def test_changed_lock_version_fails(self):
|
||||
result = self.run_check(APP + LIBC + UNTRUSTED, header="version = 3\n\n")
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("top-level 'version'", result.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,71 @@
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPT = REPO_ROOT / "ci" / "check_sdist.py"
|
||||
ROOT = "lancedb-1.0.0"
|
||||
|
||||
|
||||
class CheckSdistTest(unittest.TestCase):
|
||||
def run_check(self, tracked, shipped, links=(), root=ROOT):
|
||||
"""Run the check in a git repo tracking `tracked`, on an sdist holding `shipped`
|
||||
as files and `links` as symlinks, under the folder `root`."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
subprocess.run(["git", "init", "-q", tmp], check=True)
|
||||
for name in tracked:
|
||||
path = Path(tmp, "python/python/lancedb", name)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text("x = 1\n")
|
||||
if tracked:
|
||||
subprocess.run(["git", "-C", tmp, "add", "."], check=True)
|
||||
sdist = Path(tmp, f"{ROOT}.tar.gz")
|
||||
with tarfile.open(sdist, "w:gz") as tar:
|
||||
for name in shipped:
|
||||
member = Path(tmp, "member")
|
||||
member.write_text("x = 1\n")
|
||||
tar.add(member, arcname=f"{root}/python/lancedb/{name}")
|
||||
for name in links:
|
||||
link = tarfile.TarInfo(f"{root}/python/lancedb/{name}")
|
||||
link.type = tarfile.SYMTYPE
|
||||
link.linkname = "__init__.py"
|
||||
tar.addfile(link)
|
||||
return subprocess.run(
|
||||
[sys.executable, SCRIPT, sdist],
|
||||
cwd=tmp,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
def test_complete_sdist_passes(self):
|
||||
result = self.run_check(["__init__.py", "db.py"], ["__init__.py", "db.py"])
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
|
||||
def test_missing_tracked_file_fails(self):
|
||||
result = self.run_check(["__init__.py", "db.py"], ["__init__.py"])
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("python/python/lancedb/db.py", result.stderr)
|
||||
|
||||
def test_tracked_file_shipped_as_symlink_fails(self):
|
||||
result = self.run_check(
|
||||
["__init__.py", "db.py"], ["__init__.py"], links=["db.py"]
|
||||
)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("python/python/lancedb/db.py", result.stderr)
|
||||
|
||||
def test_files_outside_the_sdist_root_fail(self):
|
||||
result = self.run_check(["__init__.py"], ["__init__.py"], root="other-1.0.0")
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("python/python/lancedb/__init__.py", result.stderr)
|
||||
|
||||
def test_no_tracked_files_fails(self):
|
||||
result = self.run_check([], ["__init__.py"])
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("git tracks no files", result.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -80,6 +80,18 @@ ignore = [
|
||||
{ id = "RUSTSEC-2026-0176", reason = "pyo3 in Python bindings; awaiting patched pyo3 release" },
|
||||
{ id = "RUSTSEC-2026-0177", reason = "pyo3 in Python bindings; awaiting patched pyo3 release" },
|
||||
|
||||
# zip < 9.0.0: SUID/SGID bits, symlink targets and permissions applied through
|
||||
# symlinks during archive extraction. Transitive via candle-core (optional
|
||||
# local-embedding feature); every published candle release pins zip < 9
|
||||
# (0.9.x/0.10.x ^7.2, 0.11.0 ^8.6), so no dependency bump clears it.
|
||||
# LanceDB never extracts archives; the zip crate is only linked.
|
||||
# https://rustsec.org/advisories/RUSTSEC-2026-0367
|
||||
# https://rustsec.org/advisories/RUSTSEC-2026-0368
|
||||
# https://rustsec.org/advisories/RUSTSEC-2026-0369
|
||||
{ id = "RUSTSEC-2026-0367", reason = "zip via candle-core; candle pins zip < 9 and lancedb never extracts archives" },
|
||||
{ id = "RUSTSEC-2026-0368", reason = "zip via candle-core; candle pins zip < 9 and lancedb never extracts archives" },
|
||||
{ id = "RUSTSEC-2026-0369", reason = "zip via candle-core; candle pins zip < 9 and lancedb never extracts archives" },
|
||||
|
||||
# quick-xml < 0.41.0: quadratic runtime on duplicate attribute names (DoS).
|
||||
# quick-xml < 0.41.0: unbounded namespace-declaration allocation in NsReader (DoS).
|
||||
# Pulled in transitively by inferno (dev-only flame-graph dep), lance-namespace-impls
|
||||
|
||||
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
|
||||
<dependency>
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-core</artifactId>
|
||||
<version>0.41.0-beta.7</version>
|
||||
<version>0.41.0-beta.9</version>
|
||||
</dependency>
|
||||
```
|
||||
|
||||
|
||||
@@ -82,7 +82,7 @@ curl -X POST "$LANCEDB_HOST_OVERRIDE/v1/table/words/describe" \
|
||||
-d '{}'
|
||||
```
|
||||
|
||||
That's it -- you're connected! Now, you can start adding data and querying it.
|
||||
That's it. You're connected! Now, you can start adding data and querying it.
|
||||
You can visit the tutorial section to build your own applications with LanceDB.
|
||||
|
||||
<Card
|
||||
|
||||
@@ -13,7 +13,7 @@ import {
|
||||
} from '/snippets/build_with_ai_agents.mdx';
|
||||
|
||||
A coding agent writing LanceDB code from memory writes whatever was in its
|
||||
training data -- often an older API, sometimes one that never existed. The
|
||||
training data: often an older API, sometimes one that never existed. The
|
||||
LanceDB agent plugin fixes that by supplying the current reference at the point
|
||||
the agent needs it.
|
||||
|
||||
@@ -42,8 +42,8 @@ npx plugins add lancedb/lancedb-agent-plugins
|
||||
|
||||
The [`plugins`](https://www.npmjs.com/package/plugins) installer shown under
|
||||
**Other agents** is a cross-tool option. It detects which agent CLIs are on your
|
||||
`PATH` — Cursor, GitHub Copilot CLI, VS Code, Grok Build, and Kimi Code, as well
|
||||
as Claude Code and Codex — and installs through each one's native plugin system.
|
||||
`PATH` (Cursor, GitHub Copilot CLI, VS Code, Grok Build, and Kimi Code, as well
|
||||
as Claude Code and Codex) and installs through each one's native plugin system.
|
||||
|
||||
The plugin supplements the agent's training data with current LanceDB
|
||||
instructions. To pick up later revisions, refresh the marketplace:
|
||||
|
||||
@@ -108,7 +108,7 @@ A few starting points for dashboards and alerts:
|
||||
|
||||
<Badge color="red">Enterprise</Badge> A deployment emits its own metrics from
|
||||
the server side, using Prometheus for collection and OpenTelemetry (OTel) to
|
||||
export them with data enrichment -- no client-side instrumentation needed.
|
||||
export them with data enrichment. No client-side instrumentation is needed.
|
||||
|
||||
| Component | Metrics |
|
||||
| --- | --- |
|
||||
|
||||
@@ -16,7 +16,7 @@ availability:
|
||||
This page is about the embedded library, where your connection chooses and
|
||||
configures the storage backend. A LanceDB Enterprise deployment sets up its own
|
||||
storage when it is deployed, so a `db://` connection passes none of these
|
||||
options except an Azure storage account name -- see
|
||||
options except an Azure storage account name. See
|
||||
[Deployment](/enterprise/deployment) for how a deployment is set up.
|
||||
|
||||
LanceDB's storage layer is built on modular, disk-first components. That design makes it flexible enough to run across local NVMe, EBS, EFS, and any object store that exposes an S3-compatible API. It also supports region-specific backends such as [Tencent COS](/storage/configuration#tencent-cos) and cache-acceleration layers such as [GooseFS](/storage/configuration#goosefs).
|
||||
@@ -49,7 +49,7 @@ LanceDB separates storage and compute and writes immutable fragments, making it
|
||||
<Note>
|
||||
**Concurrent writers on S3**
|
||||
|
||||
S3 and S3 Express now support atomic writes natively, so LanceDB handles concurrent writers against the same table out-of-the-box — no external commit coordinator is required. Bucket-level [server-side encryption with KMS](/storage/configuration#server-side-encryption-with-kms) and [S3 Express One Zone](/storage/configuration#s3-express) are also supported on this tier.
|
||||
S3 and S3 Express now support atomic writes natively, so LanceDB handles concurrent writers against the same table out-of-the-box: no external commit coordinator is required. Bucket-level [server-side encryption with KMS](/storage/configuration#server-side-encryption-with-kms) and [S3 Express One Zone](/storage/configuration#s3-express) are also supported on this tier.
|
||||
</Note>
|
||||
|
||||
### 2\. File storage (EFS, GCS Filestore, Azure File) {#2-file-storage-efs-gcs-filestore-azure-file}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
title: "Managing Embeddings"
|
||||
sidebarTitle: "Managing Embeddings"
|
||||
description: "Use the embedding API in LanceDB -- registry, functions, schemas, and multi-language SDK support."
|
||||
description: "Use the embedding API in LanceDB: registry, functions, schemas, and multi-language SDK support."
|
||||
icon: "bars"
|
||||
availability:
|
||||
oss: available
|
||||
@@ -134,6 +134,29 @@ For non-sensitive settings such as inference device selection, you can also use
|
||||
|
||||
Find the full list of arguments for each provider in the [integrations](/integrations/embedding) section.
|
||||
|
||||
### Trusting remote code and custom endpoints {#trusting-remote-code-and-custom-endpoints}
|
||||
|
||||
In Python, the embedding definition is stored in the table, so anyone who can write to the table can change it.
|
||||
To keep a table from running code or sending data where you didn't intend, two kinds of settings are only honored
|
||||
when this process has approved them:
|
||||
|
||||
- `trust_remote_code=True`, which runs code from the model's Hugging Face repository.
|
||||
- Custom endpoints such as OpenAI `base_url`, Ollama `host`, or watsonx `url`, which receive your data and API keys.
|
||||
|
||||
Embedding functions you create in the current process are approved automatically. When you open a table in a new
|
||||
process, approve these settings explicitly before computing embeddings:
|
||||
|
||||
```python
|
||||
from lancedb.embeddings import get_registry
|
||||
|
||||
registry = get_registry()
|
||||
# Pinning a commit hash is recommended, since the code at a branch can change.
|
||||
registry.allow_remote_code("nomic-ai/nomic-embed-text-v1.5", revision="<commit-hash>")
|
||||
registry.allow_endpoint("http://my-ollama-host:11434")
|
||||
```
|
||||
|
||||
Until then, computing embeddings raises `UntrustedEmbeddingConfigError`. Other operations on the table still work.
|
||||
|
||||
## Multiple embedding columns {#multiple-embedding-columns}
|
||||
|
||||
A single table can include more than one embedding definition when you want to store multiple semantic views
|
||||
|
||||
@@ -9,7 +9,7 @@ keywords: ["architecture", "control plane", "data plane", "indexer"]
|
||||
<Badge color="red">Enterprise</Badge>
|
||||
|
||||
This page describes a LanceDB Enterprise capability. The embedded form
|
||||
of LanceDB has no server to provide it -- see
|
||||
of LanceDB has no server to provide it. See
|
||||
[Architecture](/architecture) for what differs between the two.
|
||||
|
||||
A deployment separates compute from storage so each scales on its own: table
|
||||
|
||||
@@ -17,8 +17,8 @@ A deployment authenticates every request before
|
||||
[authorization](/governance/authorization) decides what it may do, against a
|
||||
`db://` remote table, in one of two ways:
|
||||
|
||||
- **API keys** -- a long-lived shared secret passed as `api_key` on connect. Works in every SDK.
|
||||
- **OAuth 2.0** -- short-lived bearer tokens obtained from your identity provider, refreshed automatically by the client.
|
||||
- **API keys**: a long-lived shared secret passed as `api_key` on connect. Works in every SDK.
|
||||
- **OAuth 2.0**: short-lived bearer tokens obtained from your identity provider, refreshed automatically by the client.
|
||||
|
||||
OAuth is the recommended option when you want to rotate credentials centrally,
|
||||
plug into an existing identity provider, or run on Azure with managed
|
||||
@@ -48,7 +48,7 @@ An API key belongs in the environment rather than in source. See
|
||||
|
||||
The async Python client and the TypeScript client can obtain bearer tokens
|
||||
from an OIDC issuer and attach them to every request. Token acquisition,
|
||||
caching, and refresh are handled inside the client -- your application code
|
||||
caching, and refresh are handled inside the client. Your application code
|
||||
only provides the configuration.
|
||||
|
||||
A deployment accepts these tokens only when its administrator has configured
|
||||
@@ -162,7 +162,7 @@ field names; TypeScript uses `camelCase`.
|
||||
| `flow` | No | Selects the OAuth flow. Defaults to Client Credentials. |
|
||||
| `client_secret` / `clientSecret` | Conditional | Required for Client Credentials. Redacted from the config's `repr` (Python) and `Debug` output (TypeScript native binding) so it does not leak into logs. |
|
||||
| `managed_identity_client_id` / `managedIdentityClientId` | No | Client ID of a user-assigned managed identity. Only used with Azure Managed Identity; omit for system-assigned identities. |
|
||||
| `refresh_buffer_secs` / `refreshBufferSecs` | No | How many seconds before token expiry to proactively refresh. Defaults to 300. Keep this well below the token TTL -- setting it greater than or equal to the TTL forces a refresh on every request. |
|
||||
| `refresh_buffer_secs` / `refreshBufferSecs` | No | How many seconds before token expiry to proactively refresh. Defaults to 300. Keep this well below the token TTL: setting it greater than or equal to the TTL forces a refresh on every request. |
|
||||
|
||||
<Warning>
|
||||
Treat the client secret like any other production credential. Load it from a
|
||||
@@ -171,5 +171,5 @@ control. Both SDKs deliberately omit the secret from their debug/repr output
|
||||
so accidental log lines do not expose the value.
|
||||
</Warning>
|
||||
|
||||
Embedded LanceDB has nothing to authenticate to -- your process opens the files
|
||||
Embedded LanceDB has nothing to authenticate to: your process opens the files
|
||||
directly, and the object store's own credentials are the only gate.
|
||||
|
||||
@@ -9,11 +9,11 @@ keywords: ["benchmarks", "performance", "throughput", "latency"]
|
||||
<Badge color="red">Enterprise</Badge>
|
||||
|
||||
This page describes a LanceDB Enterprise capability. The embedded form
|
||||
of LanceDB has no server to provide it -- see
|
||||
of LanceDB has no server to provide it. See
|
||||
[Architecture](/architecture) for what differs between the two.
|
||||
|
||||
Benchmark results are specific to a deployment's shape -- cluster size, storage
|
||||
class, dataset -- so they are published with LanceDB Enterprise.
|
||||
Benchmark results are specific to a deployment's shape (cluster size, storage
|
||||
class, dataset), so they are published with LanceDB Enterprise.
|
||||
|
||||
For tuning your own workload rather than comparing published numbers, see
|
||||
[Query plans](/search/explain) and the performance notes on
|
||||
|
||||
@@ -9,7 +9,7 @@ keywords: ["azure", "deployment", "aks"]
|
||||
<Badge color="red">Enterprise</Badge>
|
||||
|
||||
This page describes a LanceDB Enterprise capability. The embedded form
|
||||
of LanceDB has no server to provide it -- see
|
||||
of LanceDB has no server to provide it. See
|
||||
[Architecture](/architecture) for what differs between the two.
|
||||
|
||||
The Azure deployment procedure is published with LanceDB Enterprise. See
|
||||
|
||||
@@ -9,12 +9,12 @@ keywords: ["deployment", "install", "cluster", "kubernetes"]
|
||||
<Badge color="red">Enterprise</Badge>
|
||||
|
||||
This page describes a LanceDB Enterprise capability. The embedded form
|
||||
of LanceDB has no server to provide it -- see
|
||||
of LanceDB has no server to provide it. See
|
||||
[Architecture](/architecture) for what differs between the two.
|
||||
|
||||
Deployment is specific to your cloud and account, so the procedure is published
|
||||
with LanceDB Enterprise. [Contact us](https://www.lancedb.com/contact) to get
|
||||
started.
|
||||
|
||||
Embedded LanceDB has nothing to deploy -- it is a library your process links
|
||||
Embedded LanceDB has nothing to deploy: it is a library your process links
|
||||
against. See [Architecture](/architecture#embedded).
|
||||
|
||||
@@ -9,7 +9,7 @@ keywords: ["enterprise", "deployment", "cluster"]
|
||||
<Badge color="red">Enterprise</Badge>
|
||||
|
||||
This page describes a LanceDB Enterprise capability. The embedded form
|
||||
of LanceDB has no server to provide it -- see
|
||||
of LanceDB has no server to provide it. See
|
||||
[Architecture](/architecture) for what differs between the two.
|
||||
|
||||
LanceDB Enterprise runs the lakehouse as a service: a control plane for
|
||||
|
||||
@@ -9,11 +9,11 @@ keywords: ["compliance", "security", "soc2", "encryption", "audit"]
|
||||
<Badge color="red">Enterprise</Badge>
|
||||
|
||||
This page describes a LanceDB Enterprise capability. The embedded form
|
||||
of LanceDB has no server to provide it -- see
|
||||
of LanceDB has no server to provide it. See
|
||||
[Architecture](/architecture) for what differs between the two.
|
||||
|
||||
Compliance posture -- certifications, data residency, encryption at rest and in
|
||||
transit, audit logging -- is a property of a deployment, so it is documented
|
||||
Compliance posture (certifications, data residency, encryption at rest and in
|
||||
transit, audit logging) is a property of a deployment, so it is documented
|
||||
with LanceDB Enterprise.
|
||||
|
||||
For embedded LanceDB, these are properties of the object store you point it at
|
||||
|
||||
@@ -28,7 +28,7 @@ table.add_columns(computed={"doubled": "x * 2"})
|
||||
```
|
||||
|
||||
That commits a new table version with the column present and empty. It is an
|
||||
ordinary table version, so the usual [versioning](/management/table#versioning) applies --
|
||||
ordinary table version, so the usual [versioning](/management/table#versioning) applies,
|
||||
including that you can travel back to before the column existed.
|
||||
|
||||
## Compute now or on refresh {#computed-vs-transforms}
|
||||
@@ -61,7 +61,7 @@ table.add_columns(computed={
|
||||
})
|
||||
```
|
||||
|
||||
Declaration order is dependency order -- a computed column may reference one
|
||||
Declaration order is dependency order: a computed column may reference one
|
||||
declared earlier in the same mapping, but not one declared after it.
|
||||
|
||||
The functions available are the ones the
|
||||
@@ -92,8 +92,8 @@ reshape later. Dropping the computed column releases the constraint.
|
||||
|
||||
<Badge color="red">Enterprise</Badge>
|
||||
|
||||
When the value cannot be expressed in SQL -- an embedding, a model call, an
|
||||
image decode -- [Geneva](/geneva), the feature engineering package for LanceDB
|
||||
When the value cannot be expressed in SQL (an embedding, a model call, an
|
||||
image decode), [Geneva](/geneva), the feature engineering package for LanceDB
|
||||
Enterprise, fills the column with a Python function instead.
|
||||
|
||||
{/*
|
||||
|
||||
@@ -111,7 +111,7 @@ The expressions above are evaluated once, as part of the call. For other workflo
|
||||
expression and leaves the column empty, so you choose when the work happens.
|
||||
Use it when you want to schedule the computation separately.
|
||||
- On LanceDB Enterprise, [Geneva](/geneva) fills a column with Python, for
|
||||
values SQL cannot express -- an embedding, a model call, an image decode.
|
||||
values SQL cannot express: an embedding, a model call, an image decode.
|
||||
|
||||
{/*
|
||||
Draft for the Functions release. Replaces the Geneva item in the list above:
|
||||
|
||||
@@ -24,7 +24,7 @@ then refresh the derived data when its inputs change.
|
||||
*/}
|
||||
|
||||
<Note>
|
||||
In LanceDB Enterprise, every operation here also has a SQL form -- see [DDL](/sql/ddl/table#alter) to declare a column and [REFRESH](/sql/dml/refresh) to fill it.
|
||||
In LanceDB Enterprise, every operation here also has a SQL form. See [DDL](/sql/ddl/table#alter) to declare a column and [REFRESH](/sql/dml/refresh) to fill it.
|
||||
</Note>
|
||||
|
||||
## Computed columns and materialized views {#the-shape}
|
||||
@@ -91,8 +91,8 @@ Draft for the Functions release. Replace the Geneva row of the table above with:
|
||||
**Computed columns** are the common case: a column declared with a SQL
|
||||
expression, filled by [a refresh](/feature-engineering/refresh-column).
|
||||
|
||||
**Materialized views** derive a whole table -- a filtered, projected, or
|
||||
aggregated view of a source -- and bring it up to date on each refresh. The
|
||||
**Materialized views** derive a whole table (a filtered, projected, or
|
||||
aggregated view of a source) and bring it up to date on each refresh. The
|
||||
result is an ordinary table: queryable, indexable, searchable.
|
||||
|
||||
**Geneva** is LanceDB Enterprise's Python package for work SQL cannot express:
|
||||
@@ -128,7 +128,7 @@ result.rows_assigned
|
||||
```
|
||||
|
||||
A backfill over a large table can therefore be started, left, and checked on
|
||||
from a different process -- the connection does not have to stay open.
|
||||
from a different process: the connection does not have to stay open.
|
||||
|
||||
Enterprise manages the compute for the refresh. You still control when to
|
||||
submit it; declaring a computed column does not schedule recurring refreshes.
|
||||
|
||||
@@ -31,8 +31,8 @@ REFRESH COLUMN doubled ON my_table
|
||||
```
|
||||
</CodeGroup>
|
||||
|
||||
The refresh commits ordinary table versions -- the values, then a
|
||||
metadata-only record of the inputs they were computed from -- so
|
||||
The refresh commits ordinary table versions (the values, then a
|
||||
metadata-only record of the inputs they were computed from), so
|
||||
[versioning](/management/table#versioning) applies to it like any other write.
|
||||
|
||||
<Note>
|
||||
@@ -53,7 +53,7 @@ column and then ingesting more data does not mean recomputing the backfill.
|
||||
derives from `x` and a row's `x` is later updated, `doubled` keeps its old value
|
||||
until the next refresh recomputes it.
|
||||
|
||||
That is what makes the call idempotent and safe to retry -- but unlike a
|
||||
That is what makes the call idempotent and safe to retry. However, unlike a
|
||||
spreadsheet formula, nothing is recomputed until you refresh, so a derived
|
||||
column is stale between an update and the next refresh.
|
||||
|
||||
@@ -122,8 +122,8 @@ values can remain after a failure.
|
||||
Either way, recover by fixing the cause and refreshing again. The new refresh
|
||||
fills rows that are still empty or whose inputs changed, and normally skips
|
||||
values that were kept. That is not guaranteed: a refresh can recompute values
|
||||
that were already written -- for example when the record of the inputs they
|
||||
were computed from was not saved -- so do not count on a retry costing only the
|
||||
that were already written (for example when the record of the inputs they
|
||||
were computed from was not saved), so do not count on a retry costing only the
|
||||
rows that did not complete.
|
||||
|
||||
{/*
|
||||
|
||||
@@ -23,8 +23,8 @@ A grant is held by one of four kinds of subject, distinguished by a prefix:
|
||||
|
||||
| Prefix | Subject |
|
||||
| --- | --- |
|
||||
| `p:` | a principal, by ID -- `P:` by name |
|
||||
| `g:` | a group, by ID -- `G:` by name |
|
||||
| `p:` | a principal, by ID (`P:` by name) |
|
||||
| `g:` | a group, by ID (`G:` by name) |
|
||||
| `r:` | a role |
|
||||
| `a:` | the principal holding an API key, named by the key |
|
||||
|
||||
@@ -67,7 +67,7 @@ A nested namespace joins its path with `$`, as in `namespace:db:parent$child`
|
||||
or `table:db:parent$child$table`.
|
||||
|
||||
The hierarchy is not decorative. Reaching a table requires `USAGE` on the
|
||||
namespace containing it and on the database containing that -- a grant on the
|
||||
namespace containing it and on the database containing that. A grant on the
|
||||
table alone is not enough to traverse to it.
|
||||
|
||||
## Privileges {#privileges}
|
||||
@@ -118,14 +118,14 @@ A subtlety worth knowing, because it explains grants that appear to work on
|
||||
objects that do not exist yet.
|
||||
|
||||
When you create an object, the deployment checks the database and each
|
||||
namespace that will contain it. One that **has no owner** -- because it does
|
||||
not exist yet, or predates per-object authorization -- is treated as one you are
|
||||
namespace that will contain it. One that **has no owner** (because it does
|
||||
not exist yet, or predates per-object authorization) is treated as one you are
|
||||
creating: the check falls back to `CREATE_DATABASE` on `system` for the
|
||||
database, or to `CREATE_NAMESPACE` on its parent for a namespace, and you become
|
||||
its owner. One that has an owner needs `USAGE` as usual.
|
||||
|
||||
So a principal holding `CREATE_DATABASE` can create a table in a database that
|
||||
has not been created -- which is what makes creation possible at all, and why
|
||||
has not been created, which is what makes creation possible at all, and why
|
||||
an empty ACL set is not the same as a deny. The fallback applies only when
|
||||
creating; other operations on a database or namespace need `USAGE` on it.
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ import {
|
||||
} from '/snippets/indexing.mdx';
|
||||
|
||||
As you add new data to your LanceDB tables, your indexes may become outdated.
|
||||
Reindexing is the process of updating the index to account for new data -- this applies to either a full-text search (FTS) index or a vector index. Reindexing is an important operation to run periodically as your data grows, as it has performance implications.
|
||||
Reindexing is the process of updating the index to account for new data. This applies to either a full-text search (FTS) index or a vector index. Reindexing is an important operation to run periodically as your data grows, as it has performance implications.
|
||||
|
||||
As data is being added and a reindex operation is running, LanceDB will combine results from the existing index with exhaustive/flat search on the new data. This is done to ensure that you're still retrieving results over all your data, but it does come at a performance cost. The more data that you add without reindexing, the impact on latency (due to exhaustive search) can be noticeable.
|
||||
|
||||
|
||||
@@ -40,7 +40,7 @@ LanceDB implements **ANN (Approximate Nearest-Neighbor)** queries with several t
|
||||
## Do you need one? {#do-you-need-one}
|
||||
|
||||
Not always. LanceDB's disk-based brute-force kNN search is fast even without an
|
||||
index -- computing 100K pairs of 1000-dimension vectors takes under 20ms in
|
||||
index: computing 100K pairs of 1000-dimension vectors takes under 20ms in
|
||||
benchmarks. For datasets around 100K rows or fewer, or applications that can
|
||||
accept ~100ms latency, a vector index is usually unnecessary overhead.
|
||||
|
||||
@@ -276,7 +276,7 @@ The snippet below builds and queries an `IVF_HNSW_SQ` index.
|
||||
In LanceDB Enterprise, vector indexes are managed **automatically**. The system asynchronously updates and optimizes indexes as a background process:
|
||||
- Automatically manages indexing parameters and storage
|
||||
- Can build an `IVF_PQ` index automatically on a vector column the deployment is configured to index
|
||||
- GPU-based `IVF_PQ` builds are available on request -- [contact support](/deployment/support) to enable them for your deployment. Build time depends on dataset size, dimensions and GPU resources.
|
||||
- GPU-based `IVF_PQ` builds are available on request. [Contact support](/deployment/support) to enable them for your deployment. Build time depends on dataset size, dimensions and GPU resources.
|
||||
|
||||
<Badge color="gray">Open-Source</Badge>
|
||||
LanceDB OSS users can manually create vector indexes by calling `table.create_index()`.
|
||||
@@ -350,7 +350,7 @@ discussion of LanceDB vector indexes and quantized vector indexes can be found [
|
||||
|
||||
### Quantization Techniques
|
||||
LanceDB provides $3$ distinct quantization techniques: Product Quantization (PQ), RaBitQ Quantization (RQ), and Scalar Quantization (SQ).
|
||||
Recall that all quantizations perform **lossy** compression, in that they irreversibly lose some degree of precision in order to
|
||||
Recall that all quantizations perform **lossy** compression, in that they irreversibly lose some degree of precision to
|
||||
compactly store an index.
|
||||
|
||||
#### Product Quantization (PQ)
|
||||
|
||||
@@ -51,7 +51,7 @@ On running this query, you'll get 5 results fetched from the lancedb table, wher
|
||||
Now that we've seen how you can use LanceDB in a Genkit pipeline, let's refine the flow and create a RAG. A RAG flow will consist of an index and a retriever with its outputs postprocessed and fed into an LLM for final response
|
||||
|
||||
### Creating custom indexer flows {#creating-custom-indexer-flows}
|
||||
You can also create custom indexer flows, utilizing more options and features provided by LanceDB.
|
||||
You can also create custom indexer flows, using more options and features provided by LanceDB.
|
||||
|
||||
<CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
|
||||
{TsFrameworksGenkitCustomIndexer}
|
||||
@@ -64,7 +64,7 @@ In your console, you can see the logs
|
||||
<img width="511" alt="Screenshot 2025-05-11 at 7 19 14 PM" src="https://github.com/user-attachments/assets/243f26c5-ed38-40b6-b661-002f40f0423a" />
|
||||
|
||||
### Creating custom retriever flows {#creating-custom-retriever-flows}
|
||||
You can also create custom retriever flows, utilizing more options and features provided by LanceDB.
|
||||
You can also create custom retriever flows, using more options and features provided by LanceDB.
|
||||
<CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
|
||||
{TsFrameworksGenkitCustomRetriever}
|
||||
</CodeBlock>
|
||||
|
||||
@@ -11,7 +11,7 @@ dedicated slot for external *memory providers* that run alongside its built-in n
|
||||
|
||||
The [LanceDB memory plugin](https://github.com/lancedb/hermes-agent-memory) fills that slot.
|
||||
It gives Hermes durable, semantic recall across sessions: state a preference or a project
|
||||
convention once, and the agent can retrieve it weeks later in a brand-new session — even when
|
||||
convention once, and the agent can retrieve it weeks later in a brand-new session, even when
|
||||
you ask for it in completely different words. Everything runs inside Hermes' own Python
|
||||
process, storing a single LanceDB table on local disk. There's no memory server to operate.
|
||||
|
||||
@@ -26,20 +26,20 @@ process, storing a single LanceDB table on local disk. There's no memory server
|
||||
|
||||
Out of the box, Hermes remembers with a small curated notes file frozen into the system
|
||||
prompt, plus lexical (keyword) search over past sessions. Both are useful, but keyword search
|
||||
misses paraphrases of what you originally typed — the exact thing you need when recalling a
|
||||
misses paraphrases of what you originally typed, the exact thing you need when recalling a
|
||||
fact you phrased differently months ago.
|
||||
|
||||
LanceDB OSS is an embedded retrieval library, which makes it a natural fit here:
|
||||
|
||||
- **No server to stand up** — it reads and writes a table on local disk, so the plugin ships
|
||||
- **No server to stand up**: it reads and writes a table on local disk, so the plugin ships
|
||||
as a dependency rather than a service to operate.
|
||||
- **One table holds everything** — content, metadata, and embeddings live together. A memory
|
||||
- **One table holds everything**: content, metadata, and embeddings live together. A memory
|
||||
becomes a structured row with a category, tags, timestamps, and provenance, not just a text
|
||||
blob.
|
||||
- **Query it any way you need** — vector similarity for meaning, BM25 full-text for exact
|
||||
- **Query it any way you need**: vector similarity for meaning, BM25 full-text for exact
|
||||
names and jargon, a hybrid of the two, or plain metadata filters to keep recall scoped to
|
||||
the right workspace.
|
||||
- **It scales up** — the same table abstraction carries over to larger LanceDB deployments
|
||||
- **It scales up**: the same table abstraction carries over to larger LanceDB deployments
|
||||
later, so the local setup is never a dead end.
|
||||
|
||||
## Install and activate {#install-and-activate}
|
||||
@@ -68,7 +68,7 @@ hermes plugins install lancedb/hermes-agent-memory
|
||||
</Step>
|
||||
|
||||
<Step title="Install runtime dependencies into Hermes' environment">
|
||||
Hermes loads plugins inside its own Python interpreter, so the dependencies go *there* — not
|
||||
Hermes loads plugins inside its own Python interpreter, so the dependencies go *there*, not
|
||||
into a separate virtualenv. (This interpreter is shared across profiles, so you only install
|
||||
once.)
|
||||
|
||||
@@ -83,7 +83,7 @@ is OpenAI, so set `OPENAI_API_KEY` in your environment or in `~/.hermes/.env`.
|
||||
|
||||
<Info>
|
||||
Prefer a local or non-OpenAI model? The plugin uses an OpenAI-compatible client, so you can
|
||||
point it at any compatible endpoint (OpenRouter, Ollama, vLLM, …) in your config — no code
|
||||
point it at any compatible endpoint (OpenRouter, Ollama, vLLM, …) in your config, with no code
|
||||
change needed. See [Configuration](#configuration) below.
|
||||
</Info>
|
||||
</Step>
|
||||
@@ -95,7 +95,7 @@ Switch memory on and pick this plugin:
|
||||
hermes memory setup # choose "lancedb"
|
||||
```
|
||||
|
||||
Then confirm it's actually active before you start chatting — this is the one step worth not
|
||||
Then confirm it's actually active before you start chatting. This is the one step worth not
|
||||
skipping, because Hermes quietly falls back to its built-in notes if the provider isn't set:
|
||||
|
||||
```bash
|
||||
@@ -128,7 +128,7 @@ Once activated, the agent has four tools for working with long-term memory:
|
||||
| `lancedb_forget` | Deletes safely: previews candidates first, then deletes by exact ID, so nothing disappears by accident. |
|
||||
|
||||
Beyond these tools, the plugin also captures durable facts from your conversations
|
||||
automatically — an auxiliary model distills them before context is compressed and again when a
|
||||
automatically: an auxiliary model distills them before context is compressed and again when a
|
||||
session ends, so insights survive even when the raw messages are summarized away.
|
||||
|
||||
## Walkthrough {#walkthrough}
|
||||
@@ -157,8 +157,8 @@ line below:
|
||||
|
||||
### Recall {#recall}
|
||||
|
||||
First, take Hermes' built-in notes out of the picture so recall can *only* come from LanceDB —
|
||||
the two layers run side by side otherwise, and either could answer:
|
||||
First, take Hermes' built-in notes out of the picture so recall can *only* come from LanceDB
|
||||
(the two layers run side by side otherwise, and either could answer):
|
||||
|
||||
```bash
|
||||
hermes config set memory.memory_enabled false
|
||||
@@ -179,13 +179,13 @@ differently from how you saved it:
|
||||
```
|
||||
|
||||
This is the whole point in one exchange. You asked about your *preferences* using none of the
|
||||
words you stored the fact with — no *uv*, no *pip*, no *type hints* — yet a fresh session, with
|
||||
words you stored the fact with (no *uv*, no *pip*, no *type hints*), yet a fresh session, with
|
||||
none of the earlier conversation in context, pulls it straight back. Semantic search matches on
|
||||
meaning rather than exact wording, which is precisely where keyword search falls short.
|
||||
|
||||
<Tip>
|
||||
Turn the built-in layer back on for everyday use with `hermes config set memory.memory_enabled true`
|
||||
— the two memory layers happily coexist. We only disabled it here to prove the recall came from
|
||||
Turn the built-in layer back on for everyday use with `hermes config set memory.memory_enabled true`.
|
||||
The two memory layers happily coexist. We only disabled it here to prove the recall came from
|
||||
LanceDB.
|
||||
</Tip>
|
||||
|
||||
@@ -231,7 +231,7 @@ lifecycle of a durable memory.
|
||||
|
||||
## Retrieval modes {#retrieval-modes}
|
||||
|
||||
Recall ships in `vector` mode by default — pure semantic search, which is what survives the
|
||||
Recall ships in `vector` mode by default: pure semantic search, which is what survives the
|
||||
paraphrasing you saw above. If you also need exact name or jargon matching, switch to `hybrid`
|
||||
(vector + BM25) and choose how the two legs are fused: RRF, a vector-biased linear blend, or a
|
||||
cross-encoder reranker. Mode is set per call; fusion is a config setting.
|
||||
@@ -256,8 +256,8 @@ defaults to the compact 17M-parameter [ettin reranker](https://huggingface.co/cr
|
||||
## Inspect the store {#inspect-the-store}
|
||||
|
||||
Everything lives in one table named `memories` at `~/.hermes/lancedb/memories.lance`. Because
|
||||
it's a plain LanceDB table, you can open it directly and see exactly what the agent has stored
|
||||
— a `kind` column separates extracted `fact` rows from the raw `turn` rows they were drawn
|
||||
it's a plain LanceDB table, you can open it directly and see exactly what the agent has stored.
|
||||
A `kind` column separates extracted `fact` rows from the raw `turn` rows they were drawn
|
||||
from:
|
||||
|
||||
```python
|
||||
@@ -270,7 +270,7 @@ print(tbl.to_pandas()[["kind", "category", "content"]].head())
|
||||
|
||||
## Configuration {#configuration}
|
||||
|
||||
The plugin runs on sensible defaults once activated — you don't have to configure anything.
|
||||
The plugin runs on sensible defaults once activated: you don't have to configure anything.
|
||||
`~/.hermes/config.yaml` is purely for overrides. Two common ones:
|
||||
|
||||
Use a cheaper model for the auxiliary fact-extraction calls:
|
||||
@@ -298,7 +298,7 @@ plugins:
|
||||
|
||||
<Info>
|
||||
Changing the embedding model (or its dimension) against an existing store requires recreating
|
||||
the table — the plugin fails loudly on a dimension mismatch rather than silently returning
|
||||
the table. The plugin fails loudly on a dimension mismatch rather than silently returning
|
||||
nothing. Every option is documented in the plugin's [`default_config.yaml`](https://github.com/lancedb/hermes-agent-memory/blob/main/src/default_config.yaml).
|
||||
</Info>
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ Kiln's [app](https://kiln.tech/download) makes it easy to:
|
||||
|
||||
## Find the Best RAG Pipeline for Your Use Case {#find-the-best-rag-pipeline-for-your-use-case}
|
||||
|
||||
There is no universal best RAG solution—only the best solution for your specific use case. Kiln makes it easy to compare state-of-the-art configurations and find which works best for you.
|
||||
There is no universal best RAG solution, only the best solution for your specific use case. Kiln makes it easy to compare state-of-the-art configurations and find which works best for you.
|
||||
|
||||
Start with pre-configured templates for state-of-the-art RAG at various performance/quality/cost levels, or experiment with any combination of options:
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ import {
|
||||
|
||||
LangChain streamlines these stages (in figure above) by providing pre-built components and tools for integration, memory management, and deployment, allowing developers to focus on application logic rather than underlying complexities.
|
||||
|
||||
Integration of **Langchain** with **LanceDB** enables applications to retrieve the most relevant data by comparing query vectors against stored vectors, facilitating effective information retrieval. It results in better and context aware replies and actions by the LLMs.
|
||||
Integration of **LangChain** with **LanceDB** enables applications to retrieve the most relevant data by comparing query vectors against stored vectors, facilitating effective information retrieval. It results in better and context aware replies and actions by the LLMs.
|
||||
|
||||
## Quick Start {#quick-start}
|
||||
You can load your document data using langchain's loaders, for this example we are using `TextLoader` and `OpenAIEmbeddings` as the embedding model.
|
||||
@@ -42,7 +42,7 @@ The exhaustive list of parameters for `LanceDB` vector store are :
|
||||
|Name|type|Purpose|default|
|
||||
|:----|:----|:----|:----|
|
||||
|`connection`| (Optional) `Any` |`lancedb.db.LanceDBConnection` connection object to use. If not provided, a new connection will be created.|`None`|
|
||||
|`embedding`| (Optional) `Embeddings` | Langchain embedding model.|Provided by user.|
|
||||
|`embedding`| (Optional) `Embeddings` | LangChain embedding model.|Provided by user.|
|
||||
|`uri`| (Optional) `str` |It specifies the directory location of **LanceDB database** and establishes a connection that can be used to interact with the database. |`/tmp/lancedb`|
|
||||
|`vector_key` |(Optional) `str`| Column name to use for vector's in the table.|`'vector'`|
|
||||
|`id_key` |(Optional) `str`| Column name to use for id's in the table.|`'id'`|
|
||||
@@ -51,10 +51,10 @@ The exhaustive list of parameters for `LanceDB` vector store are :
|
||||
|`api_key` |(Optional `str`) |API key to use for LanceDB Enterprise deployment.|`None`|
|
||||
|`region` |(Optional) `str`| Region to use for LanceDB Enterprise deployment.| Only for LanceDB Enterprise : `None`.|
|
||||
|`mode` |(Optional) `str` | Mode to use for adding data to the table. Valid values are "append" and "overwrite".|`'overwrite'`|
|
||||
|`table`| (Optional) `Any`| You can connect to an existing table of LanceDB, created outside of langchain, and utilize it.|`None`|
|
||||
|`table`| (Optional) `Any`| You can connect to an existing table of LanceDB, created outside of LangChain, and use it.|`None`|
|
||||
|`distance`|(Optional) `str`| The choice of distance metric used to calculate the similarity between vectors.|`'l2'`|
|
||||
|`reranker` |(Optional) `Any`| The reranker to use for LanceDB.|`None`|
|
||||
|`relevance_score_fn` |(Optional) `Callable[[float], float]` | Langchain relevance score function to be used.|`None`|
|
||||
|`relevance_score_fn` |(Optional) `Callable[[float], float]` | LangChain relevance score function to be used.|`None`|
|
||||
|`limit`|`int`|Set the maximum number of results to return.| `DEFAULT_K` (it is 4)|
|
||||
|
||||
<CodeBlock filename="Python" language="Python" icon="python">
|
||||
|
||||
@@ -40,7 +40,7 @@ In this example, we will be fetching movie information from the [Open Movie Data
|
||||
|
||||
3. **Specify necessary credentials and/or embedding model details:**
|
||||
|
||||
In order to fetch data from the OMDb API, you will need to pass a valid API key into your pipeline. Depending on whether you're using LanceDB OSS or LanceDB Enterprise, you also may need to provide the necessary credentials to connect to the LanceDB instance. These can be pasted inside `.dlt/secrets.toml`.
|
||||
To fetch data from the OMDb API, you will need to pass a valid API key into your pipeline. Depending on whether you're using LanceDB OSS or LanceDB Enterprise, you also may need to provide the necessary credentials to connect to the LanceDB instance. These can be pasted inside `.dlt/secrets.toml`.
|
||||
|
||||
dlt's LanceDB integration also allows you to automatically embed the data during ingestion. Depending on the embedding model chosen, you may need to paste the necessary credentials inside `.dlt/secrets.toml`:
|
||||
```toml
|
||||
|
||||
@@ -37,7 +37,7 @@ You can then sort your samples by similarity or use this information to find pot
|
||||
|
||||
We'll be doing the following :
|
||||
|
||||
1. **Create Index** - In order to run similarity queries against our media, we need to **index** the data. We can do this via the `compute_similarity()` function.
|
||||
1. **Create Index** - To run similarity queries against our media, we need to **index** the data. We can do this via the `compute_similarity()` function.
|
||||
- In the function, specify the **model** you want to use to generate the embedding vectors, and what **vector search engine** you want to use on the **backend** (here LanceDB).
|
||||
<Tip>
|
||||
You can also give the similarity index a name(`brain_key`), which is useful if you want to run vector searches against multiple indexes.
|
||||
|
||||
@@ -5,7 +5,7 @@ sidebarTitle: Hugging Face
|
||||
|
||||
import { PyEmbeddingHuggingfaceUsage } from '/snippets/integrations.mdx';
|
||||
|
||||
We offer support for all Hugging Face models (which can be loaded via [transformers](https://huggingface.co/docs/transformers/en/index) library). The default model is `colbert-ir/colbertv2.0` which also has its own special callout - `registry.get("colbert")`. Some Hugging Face models might require custom models defined on the HuggingFace Hub in their own modeling files. You may enable this by setting `trust_remote_code=True`. This option should only be set to True for repositories you trust and in which you have read the code, as it will execute code present on the Hub on your local machine.
|
||||
We offer support for all Hugging Face models (which can be loaded via [transformers](https://huggingface.co/docs/transformers/en/index) library). The default model is `colbert-ir/colbertv2.0` which also has its own special callout - `registry.get("colbert")`. Some Hugging Face models might require custom models defined on the Hugging Face Hub in their own modeling files. You may enable this by setting `trust_remote_code=True`. This option should only be set to True for repositories you trust and in which you have read the code, as it will execute code present on the Hub on your local machine. Pin `revision` to a commit hash, and note that tables using such a model need it approved with `get_registry().allow_remote_code(...)` when opened in a new process. See [Trusting remote code and custom endpoints](/embedding#trusting-remote-code-and-custom-endpoints).
|
||||
|
||||
Example usage:
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@ Sentence transformer embeddings are normalized by default. It is recommended to
|
||||
</Info>
|
||||
|
||||
<Warning>
|
||||
The `trust_remote_code` parameter defaults to `True`, which allows models to execute arbitrary code from their Hugging Face repository during loading. If you are loading untrusted models, set `trust_remote_code=False` to prevent remote code execution.
|
||||
Setting `trust_remote_code=True` allows the model to execute arbitrary code from its Hugging Face repository during loading. Only enable it for repositories you trust, and pin `revision` to a commit hash. When a table using such a model is opened in a new process, the model must be approved with `get_registry().allow_remote_code(...)`. See [Trusting remote code and custom endpoints](/embedding#trusting-remote-code-and-custom-endpoints).
|
||||
</Warning>
|
||||
|
||||
| Parameter | Type | Default Value | Description |
|
||||
@@ -20,7 +20,8 @@ The `trust_remote_code` parameter defaults to `True`, which allows models to exe
|
||||
| `name` | `str` | `all-MiniLM-L6-v2` | The name of the model |
|
||||
| `device` | `str` | `cpu` | The device to run the model on (can be `cpu` or `gpu`) |
|
||||
| `normalize` | `bool` | `True` | Whether to normalize the input text before feeding it to the model |
|
||||
| `trust_remote_code` | `bool` | `True` | Whether to trust and execute remote code from the model's Huggingface repository |
|
||||
| `trust_remote_code` | `bool` | `False` | Whether to trust and execute remote code from the model's Hugging Face repository |
|
||||
| `revision` | `str` | `None` | The model revision to load, such as a commit hash |
|
||||
|
||||
|
||||
<Details title="Check out available sentence-transformer models here!">
|
||||
@@ -165,5 +166,5 @@ Here is an example that uses the BAAI embedding model from the Hugging Face Hub
|
||||
</CodeBlock>
|
||||
</CodeGroup>
|
||||
</Note>
|
||||
Visit sentence-transformers [HuggingFace HUB](https://huggingface.co/sentence-transformers) page for more information on the available models.
|
||||
Visit the sentence-transformers [Hugging Face Hub](https://huggingface.co/sentence-transformers) page for more information on the available models.
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
---
|
||||
title: VoyageAI
|
||||
sidebarTitle: VoyageAI
|
||||
title: Voyage AI
|
||||
sidebarTitle: Voyage AI
|
||||
---
|
||||
|
||||
import { PyEmbeddingVoyageaiUsage, PyEmbeddingVoyageaiMultimodal } from '/snippets/integrations.mdx';
|
||||
@@ -9,7 +9,7 @@ Voyage AI provides cutting-edge embedding and rerankers.
|
||||
|
||||
|
||||
Using voyageai API requires voyageai package, which can be installed using `pip install voyageai`. Voyage AI embeddings are used to generate embeddings for text data. The embeddings can be used for various tasks like semantic search, clustering, and classification.
|
||||
You also need to set the `VOYAGE_API_KEY` environment variable to use the VoyageAI API.
|
||||
You also need to set the `VOYAGE_API_KEY` environment variable to use the Voyage AI API.
|
||||
|
||||
```shell
|
||||
export VOYAGE_API_KEY="your-api-key"
|
||||
@@ -32,7 +32,7 @@ Supported models are:
|
||||
- voyage-multimodal-3.5 (multimodal - supports text, images, and video)
|
||||
|
||||
<Info>
|
||||
**Multimodal Model:** `voyage-multimodal-3.5` supports text, images, and video inputs. It outputs 1024-dimensional embeddings by default, configurable via the `output_dimension` parameter (256, 512, 1024, 2048). See the [VoyageAI multimodal embeddings documentation](https://docs.voyageai.com/docs/multimodal-embeddings) for more details.
|
||||
**Multimodal Model:** `voyage-multimodal-3.5` supports text, images, and video inputs. It outputs 1024-dimensional embeddings by default, configurable via the `output_dimension` parameter (256, 512, 1024, 2048). See the [Voyage AI multimodal embeddings documentation](https://docs.voyageai.com/docs/multimodal-embeddings) for more details.
|
||||
</Info>
|
||||
|
||||
Supported parameters (to be passed in `create` method) are:
|
||||
|
||||
@@ -11,7 +11,7 @@ import { PyRerankingAnswerdotaiUsage } from '/snippets/integrations.mdx';
|
||||
|
||||
This integration uses [AnswersDotAI's rerankers](https://github.com/AnswerDotAI/rerankers) to rerank the search results, providing a lightweight, low-dependency, unified API to use all common reranking and cross-encoder models.
|
||||
|
||||
> **Note:** Supported query types – Hybrid, Vector, and FTS.
|
||||
> **Note:** Supported query types: Hybrid, Vector, and FTS.
|
||||
|
||||
|
||||
<CodeGroup>
|
||||
|
||||
@@ -12,7 +12,7 @@ import { PyRerankingCohereUsage } from '/snippets/provider_examples.mdx';
|
||||
This reranker uses the [Cohere](https://cohere.ai/) API to rerank the search results. You can use this reranker by passing `CohereReranker()` to the `rerank()` method. Note that you'll either need to set the `COHERE_API_KEY` environment variable or pass the `api_key` argument to use this reranker.
|
||||
|
||||
|
||||
> **Note:** Supported query types – Hybrid, Vector, and FTS.
|
||||
> **Note:** Supported query types: Hybrid, Vector, and FTS.
|
||||
|
||||
```shell
|
||||
pip install cohere
|
||||
|
||||
@@ -10,7 +10,7 @@ import { PyRerankingColbertUsage } from '/snippets/integrations.mdx';
|
||||
# ColBERT Reranker
|
||||
|
||||
This reranker uses ColBERT model to rerank the search results. You can use this reranker by passing `ColbertReranker()` to the `rerank()` method.
|
||||
> **Note:** Supported query types – Hybrid, Vector, and FTS.
|
||||
> **Note:** Supported query types: Hybrid, Vector, and FTS.
|
||||
|
||||
|
||||
<CodeGroup>
|
||||
|
||||
@@ -12,7 +12,7 @@ import { PyRerankingJinaUsage } from '/snippets/provider_examples.mdx';
|
||||
This reranker uses the [Jina](https://jina.ai/reranker/) API to rerank the search results. You can use this reranker by passing `JinaReranker()` to the `rerank()` method. Note that you'll either need to set the `JINA_API_KEY` environment variable or pass the `api_key` argument to use this reranker.
|
||||
|
||||
|
||||
> **Note:** Supported query types – Hybrid, Vector, and FTS.
|
||||
> **Note:** Supported query types: Hybrid, Vector, and FTS.
|
||||
|
||||
|
||||
Replace the credential placeholders with your provider credentials. In application code, load credentials from environment variables or a secrets manager.
|
||||
|
||||
@@ -10,7 +10,7 @@ import { PyRerankingOpenaiUsage } from '/snippets/provider_examples.mdx';
|
||||
# OpenAI Reranker (Experimental)
|
||||
|
||||
This reranker uses OpenAI chat model to rerank the search results. You can use this reranker by passing `OpenAI()` to the `rerank()` method.
|
||||
> **Note:** Supported query types – Hybrid, Vector, and FTS.
|
||||
> **Note:** Supported query types: Hybrid, Vector, and FTS.
|
||||
|
||||
> **Warning:** This reranker is experimental. OpenAI does not have a dedicated reranking model, so it uses a chat model under the hood.
|
||||
|
||||
|
||||
@@ -1,20 +1,20 @@
|
||||
---
|
||||
title: VoyageAI Reranker
|
||||
title: Voyage AI Reranker
|
||||
sidebarTitle: "Voyage AI"
|
||||
description: Integrate VoyageAI's cutting-edge reranking models with LanceDB. Features model selection, API key management, and comprehensive scoring options for all search types.
|
||||
description: Integrate Voyage AI's cutting-edge reranking models with LanceDB. Features model selection, API key management, and comprehensive scoring options for all search types.
|
||||
|
||||
---
|
||||
|
||||
import { PyRerankingVoyageaiUsage } from '/snippets/provider_examples.mdx';
|
||||
|
||||
# VoyageAI Reranker
|
||||
# Voyage AI Reranker {#voyageai-reranker}
|
||||
|
||||
Voyage AI provides cutting-edge embedding and rerankers.
|
||||
|
||||
This reranker uses the [VoyageAI](https://docs.voyageai.com/docs/) API to rerank the search results. You can use this reranker by passing `VoyageAIReranker()` to the `rerank()` method. Note that you'll either need to set the `VOYAGE_API_KEY` environment variable or pass the `api_key` argument to use this reranker.
|
||||
This reranker uses the [Voyage AI](https://docs.voyageai.com/docs/) API to rerank the search results. You can use this reranker by passing `VoyageAIReranker()` to the `rerank()` method. Note that you'll either need to set the `VOYAGE_API_KEY` environment variable or pass the `api_key` argument to use this reranker.
|
||||
|
||||
|
||||
> **Note:** Supported query types – Hybrid, Vector, and FTS.
|
||||
> **Note:** Supported query types: Hybrid, Vector, and FTS.
|
||||
|
||||
|
||||
Replace `your-api-key` with your provider API key. In application code, load
|
||||
|
||||
@@ -11,7 +11,7 @@ import { PyRerankingWatsonxUsage } from '/snippets/provider_examples.mdx';
|
||||
|
||||
This reranker uses the [IBM watsonx.ai](https://cloud.ibm.com/docs/apis/watsonx-ai#text-rerank) text rerank API to reorder search results. Pass `WatsonxReranker()` to the `rerank()` method on a query. Credentials come from the `WATSONX_API_KEY` and `WATSONX_PROJECT_ID` (or `WATSONX_SPACE_ID`) environment variables, or can be passed explicitly as arguments.
|
||||
|
||||
> **Note:** Supported query types – Hybrid, Vector, and FTS.
|
||||
> **Note:** Supported query types: Hybrid, Vector, and FTS.
|
||||
|
||||
```shell
|
||||
pip install ibm-watsonx-ai
|
||||
|
||||
@@ -12,9 +12,9 @@ availability:
|
||||
cluster configuration.
|
||||
---
|
||||
|
||||
The jobs documented elsewhere in these docs -- [Build & Repair](/indexing/reindexing),
|
||||
The jobs documented elsewhere in these docs ([Build & Repair](/indexing/reindexing),
|
||||
[Compaction](/maintenance/compaction), [Cleanup](/maintenance/cleanup),
|
||||
[Column Refresh](/feature-engineering/refresh-column) -- start from a call you
|
||||
[Column Refresh](/feature-engineering/refresh-column)) start from a call you
|
||||
make or, for compaction, cleanup and index updates, from the deployment's own
|
||||
maintenance. This page is the rest: jobs the deployment runs to keep itself
|
||||
correct and fast, which you may see named in [`SHOW JOBS`](/sql/ddl/job#show)
|
||||
@@ -32,14 +32,14 @@ log from growing without bound:
|
||||
|
||||
| Job type | What it does |
|
||||
| --- | --- |
|
||||
| `sstable_compaction` | Folds a table shard's SSTables into the base table by primary key. Distinct from `compact`, which merges base-table fragments -- the two are unrelated. |
|
||||
| `sstable_compaction` | Folds a table shard's SSTables into the base table by primary key. Distinct from `compact`, which merges base-table fragments. The two are unrelated. |
|
||||
| `wal_index_catchup` | Extends a base-table index over the rows `sstable_compaction` just folded in, and records how far the index has caught up. Nothing else advances that record. |
|
||||
|
||||
## Index maintenance {#index}
|
||||
|
||||
| Job type | What it does |
|
||||
| --- | --- |
|
||||
| `index_remap` | Reassigns index entries after row ids shift underneath an index -- the kind of remap a delta compaction can require. |
|
||||
| `index_remap` | Reassigns index entries after row ids shift underneath an index (the kind of remap a delta compaction can require). |
|
||||
| `spfresh_merge` | Merges SPFresh-managed segments for indexes that use it. |
|
||||
|
||||
## Cache warming {#cache}
|
||||
@@ -58,7 +58,7 @@ log from growing without bound:
|
||||
| `job_events_export` | Copies terminal jobs' event histories into a curated system table for metering. Registry-scoped, like the archive sweep. |
|
||||
|
||||
These two are why an old job you completed weeks ago no longer shows in
|
||||
`SHOW JOBS` -- it was archived, not deleted. Where the deployment enables the
|
||||
`SHOW JOBS`: it was archived, not deleted. Where the deployment enables the
|
||||
export, its events also survive in the exported table.
|
||||
|
||||
## Object teardown {#teardown}
|
||||
|
||||
@@ -13,11 +13,11 @@ availability:
|
||||
|
||||
Every write keeps the version before it, which is what makes
|
||||
[time travel](/management/table#versioning) possible. Cleanup is what ends
|
||||
that -- pruning versions older than a retention window, and the files that only
|
||||
that: pruning versions older than a retention window, and the files that only
|
||||
those versions referenced.
|
||||
|
||||
<Note>
|
||||
**Enterprise job type:** `cleanup`. Distinct from `compact` -- compaction
|
||||
**Enterprise job type:** `cleanup`. Distinct from `compact`: compaction
|
||||
merges files, cleanup removes them. `optimize()` in OSS runs both in one call;
|
||||
Enterprise schedules them as separate jobs, and runs cleanup only when the
|
||||
deployment's configuration enables it.
|
||||
@@ -100,7 +100,7 @@ cleanup jobs never set it.
|
||||
|
||||
<Warning>
|
||||
`delete_unverified=True` removes them regardless of age. Only set it when you
|
||||
can guarantee **no other process is writing to the table** -- otherwise it can
|
||||
can guarantee **no other process is writing to the table**. Otherwise it can
|
||||
delete files belonging to a write that is genuinely in progress, and corrupt
|
||||
the table.
|
||||
</Warning>
|
||||
@@ -109,7 +109,7 @@ the table.
|
||||
|
||||
<CardGroup cols={2}>
|
||||
<Card title="Compaction" icon="broom" href="/maintenance/compaction">
|
||||
Merging small files -- the other half of `optimize()`.
|
||||
Merging small files, the other half of `optimize()`.
|
||||
</Card>
|
||||
<Card title="Background jobs" icon="gears" href="/maintenance/background-jobs">
|
||||
The Enterprise jobs that run without a call from you.
|
||||
|
||||
@@ -34,8 +34,8 @@ Three operations, in one call:
|
||||
| | |
|
||||
| --- | --- |
|
||||
| **Compaction** | merges small files into larger ones |
|
||||
| **Prune** | removes versions older than the retention window -- see [Cleanup](/maintenance/cleanup) |
|
||||
| **Index** | folds newly added rows into existing indexes -- see [Build & Repair](/indexing/reindexing) |
|
||||
| **Prune** | removes versions older than the retention window (see [Cleanup](/maintenance/cleanup)) |
|
||||
| **Index** | folds newly added rows into existing indexes (see [Build & Repair](/indexing/reindexing)) |
|
||||
|
||||
Compaction itself is only the first of the three: merging small fragments back
|
||||
into larger ones, which is what keeps a scan from opening more files than it
|
||||
@@ -43,7 +43,7 @@ needs to.
|
||||
|
||||
<Note>
|
||||
**Enterprise job type:** `compact`. Runs automatically as data accumulates,
|
||||
separately from [`cleanup`](/maintenance/cleanup) -- the two are distinct jobs
|
||||
separately from [`cleanup`](/maintenance/cleanup). The two are distinct jobs
|
||||
even though `optimize()` bundles both in one OSS call.
|
||||
</Note>
|
||||
|
||||
|
||||
@@ -24,7 +24,7 @@ import { PyConnectEnterprisePlaceholders as FunctionConnect } from '/snippets/co
|
||||
|
||||
Some columns cannot be written as a SQL expression: an embedding, a model call,
|
||||
an image decode. A function is a Python callable registered with the database,
|
||||
versioned, and then applied as a column — the code runs where the data is
|
||||
versioned, and then applied as a column. The code runs where the data is
|
||||
rather than pulling rows to your process.
|
||||
|
||||
Registering and running a function needs a LanceDB Enterprise deployment with
|
||||
@@ -61,7 +61,7 @@ your process, without the database, so unit tests stay ordinary tests:
|
||||
</CodeBlock>
|
||||
|
||||
Where annotations cannot express the Arrow type precisely, pass `input_schema`
|
||||
and `output_schema` together — one without the other is rejected. A scalar
|
||||
and `output_schema` together: one without the other is rejected. A scalar
|
||||
output must be non-nullable, because a physical NULL is what marks a
|
||||
computed-column row as unfilled. A named-struct output may declare nullable
|
||||
fields: whether a row has a result at all is the struct's own validity, which
|
||||
@@ -181,7 +181,7 @@ pydantic**.
|
||||
<Note>
|
||||
The client version defaults `python_version`, so a function registered from a
|
||||
3.11 environment runs on 3.11 unless you say otherwise. That default is usually
|
||||
right and occasionally surprising -- pin it when the function is sensitive to it.
|
||||
right and occasionally surprising. Pin it when the function is sensitive to it.
|
||||
</Note>
|
||||
|
||||
## Failure handling {#failures}
|
||||
@@ -200,8 +200,8 @@ count on the retry costing only the rows that did not complete.
|
||||
|
||||
<Warning>
|
||||
The artifact is a **snapshot**: the function source, plus exactly the
|
||||
module-level names it references — modules as imports, importable classes and
|
||||
functions as imports, literals inlined.
|
||||
module-level names it references (modules as imports, importable classes and
|
||||
functions as imports, literals inlined).
|
||||
|
||||
Code that reaches the module namespace another way is rejected where it can be
|
||||
seen and unsupported where it cannot: `globals()`, `eval`, `sys.modules`,
|
||||
|
||||
@@ -118,7 +118,7 @@ See [Jobs](/basics/jobs).
|
||||
| drop | `drop_index` | `dropIndex` | `DROP INDEX` |
|
||||
|
||||
<Note>
|
||||
SQL `CREATE INDEX` covers one column and takes no user-chosen name -- the server
|
||||
SQL `CREATE INDEX` covers one column and takes no user-chosen name: the server
|
||||
names it. For a multi-column or named index, use the SDK.
|
||||
</Note>
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ A materialized view is a table whose contents are defined by a query over
|
||||
another table. The query is recorded in the view's schema, and a refresh
|
||||
brings its stored rows up to date with the source.
|
||||
|
||||
The result is an ordinary table -- queryable, indexable, searchable, and present
|
||||
The result is an ordinary table: queryable, indexable, searchable, and present
|
||||
in `table_names()`. Nothing about reading it is special.
|
||||
|
||||
This page covers creating and managing one. Keeping it current is
|
||||
@@ -67,7 +67,7 @@ views. On Enterprise, it asks the server for the view listing, as
|
||||
</Note>
|
||||
|
||||
A view is dropped with `drop_table`, because a materialized view **is** a table.
|
||||
That also means dropping one removes stored rows -- unlike dropping a
|
||||
That also means dropping one removes stored rows, unlike dropping a
|
||||
[view](/management/view), which removes only a saved query.
|
||||
|
||||
## Read the view and its definition {#working}
|
||||
|
||||
@@ -253,7 +253,7 @@ Below, we list some best practices for working with namespaces:
|
||||
- For simple use cases and single, stand-alone applications, the directory-based root namespace is sufficient and requires no special configuration.
|
||||
- For remote storage locations, introduce explicit namespaces when multiple teams, environments, or domains share the same catalog.
|
||||
- Treat namespace paths as stable identifiers (for example `"prod/search"`, `"staging/recs"`).
|
||||
- For maintainability reasons, avoid hard-coding object-store table paths in application code -- instead, prefer catalog identifiers + namespaces.
|
||||
- For maintainability reasons, avoid hard-coding object-store table paths in application code. Instead, prefer catalog identifiers + namespaces.
|
||||
|
||||
|
||||
## Table operations with namespace paths {#table-operations-with-namespace-paths}
|
||||
|
||||
@@ -215,7 +215,7 @@ Renaming is not supported on a local directory connection. In LanceDB Enterprise
|
||||
whether a table can be renamed or cloned at all depends on how the deployment
|
||||
stores its catalog.
|
||||
|
||||
A **shallow** clone -- the default -- references the source's data files rather
|
||||
A **shallow** clone (the default) references the source's data files rather
|
||||
than copying them, so it is fast and cheap regardless of table size. It is the
|
||||
practical way to branch an experiment off a known version, or to pin a training
|
||||
set to the exact version a model saw.
|
||||
@@ -286,7 +286,7 @@ returned job has already finished.
|
||||
|
||||
## Data types {#data-types}
|
||||
|
||||
A table's schema is an **Arrow schema** -- there is no separate LanceDB type
|
||||
A table's schema is an **Arrow schema**: there is no separate LanceDB type
|
||||
system mapped onto storage, so a column is the same object to the query engine
|
||||
as it is to the format.
|
||||
|
||||
@@ -298,7 +298,7 @@ as it is to the format.
|
||||
| Nested | `struct`, `list`, `large_list`, `fixed_size_list`, `map` |
|
||||
| Lance extensions | vectors, [JSON](/sql/data-types#json), [blobs](#multimodal), bfloat16 |
|
||||
|
||||
A vector column is a `fixed_size_list<float32, dim>` -- an ordinary Arrow type,
|
||||
A vector column is a `fixed_size_list<float32, dim>`, an ordinary Arrow type,
|
||||
which is why it is selectable and joinable but why arithmetic over it is list
|
||||
arithmetic rather than scalar.
|
||||
|
||||
@@ -307,7 +307,7 @@ extension types and pseudo-columns.
|
||||
|
||||
## Multimodal data {#multimodal}
|
||||
|
||||
Binary data -- images, audio, video, model artifacts -- lives in the table
|
||||
Binary data (images, audio, video, model artifacts) lives in the table
|
||||
beside the structured columns and embeddings that describe it, and is governed
|
||||
by the same versioning and search.
|
||||
|
||||
@@ -520,7 +520,7 @@ precision does.
|
||||
|
||||
## Schema evolution {#schema}
|
||||
|
||||
Schema evolution enables non-breaking modifications to a database table's structure — such as adding columns, altering data types, or dropping fields — to adapt to evolving data requirements without service interruptions.
|
||||
Schema evolution enables non-breaking modifications to a database table's structure (such as adding columns, altering data types, or dropping fields) to adapt to evolving data requirements without service interruptions.
|
||||
LanceDB supports ACID-compliant schema evolution through granular operations (add/alter/drop columns), allowing you to:
|
||||
|
||||
* Iterate Safely: Modify schemas in production with versioned datasets and backward compatibility
|
||||
@@ -575,7 +575,7 @@ First, let's create a sample table with product data to demonstrate schema evolu
|
||||
</CodeGroup>
|
||||
|
||||
<Note>
|
||||
Columns derived from existing data -- SQL expressions over other columns -- are
|
||||
Columns derived from existing data (SQL expressions over other columns) are
|
||||
covered in [Data evolution](/feature-engineering/data-evolution), because
|
||||
deriving values is a feature-engineering job rather than a schema one.
|
||||
</Note>
|
||||
@@ -776,7 +776,7 @@ Changing data types requires rewriting the column data and may be resource-inten
|
||||
|
||||
### Update field metadata {#schema-update-field-metadata}
|
||||
|
||||
Each column in a LanceDB table can carry a small key/value map of Arrow field metadata — useful
|
||||
Each column in a LanceDB table can carry a small key/value map of Arrow field metadata, which is useful
|
||||
for annotating columns with units, provenance, PII flags, embedding model versions, or any other
|
||||
schema-level context your application needs.
|
||||
|
||||
@@ -788,7 +788,7 @@ the new `version`.
|
||||
|
||||
Each update targets one field by **dot-path**: top-level columns are addressed by name (for
|
||||
example `"embedding"`), and nested fields by their full path (for example `"address.zip"`). By
|
||||
default, the keys you pass are **merged** into the field's existing metadata — keys you do not
|
||||
default, the keys you pass are **merged** into the field's existing metadata: keys you do not
|
||||
mention are preserved, and passing `None` (Python) or `null` (TypeScript) deletes a key. Set
|
||||
`replace: true` to swap the field's entire metadata map instead of merging.
|
||||
|
||||
@@ -823,8 +823,8 @@ To overwrite a field's metadata entirely instead of merging, set `replace` to `t
|
||||
</CodeGroup>
|
||||
|
||||
<Tip>
|
||||
You can pass multiple updates in a single call to change metadata on several fields at once —
|
||||
each call commits a single new table version.
|
||||
You can pass multiple updates in a single call to change metadata on several fields at once.
|
||||
Each call commits a single new table version.
|
||||
</Tip>
|
||||
|
||||
#### Canonical metadata keys {#schema-canonical-metadata-keys}
|
||||
@@ -908,7 +908,7 @@ Remove several columns at once for efficiency:
|
||||
<Warning title="Dropping a column is undone only by restoring a version">
|
||||
Dropping a column changes the latest version only. Until
|
||||
[cleanup](/maintenance/cleanup) prunes the versions from before the drop,
|
||||
[restoring](#versioning) one of them brings the column and its values back --
|
||||
[restoring](#versioning) one of them brings the column and its values back,
|
||||
along with the rest of the table as it was then. Once those versions are
|
||||
pruned, the drop cannot be undone, so check before dropping a column you may
|
||||
need.
|
||||
@@ -1071,11 +1071,11 @@ Numeric table versions like `v3` or `v17` are precise but hard to remember. Tags
|
||||
let you attach human-readable labels (e.g., `"prod"`, `"baseline"`,
|
||||
`"q3-evaluation"`) to specific versions and check those out by name. They are
|
||||
conceptually similar to git tags, and unlike numeric versions, **tagged versions
|
||||
are never pruned** -- a tagged version that is older than the retention window and
|
||||
are never pruned**. A tagged version that is older than the retention window and
|
||||
is not the latest stops cleanup instead (see the cleanup note at the bottom of this
|
||||
page).
|
||||
|
||||
The tags API supports the standard CRUD operations — create, list, update, delete —
|
||||
The tags API supports the standard CRUD operations (create, list, update, delete),
|
||||
plus checking out by tag name.
|
||||
|
||||
<CodeGroup>
|
||||
@@ -1099,7 +1099,7 @@ deletion, the underlying table version becomes eligible for cleanup again.
|
||||
|
||||
### Branches {#versioning-branches}
|
||||
|
||||
Beyond linear history, LanceDB also supports **branches** — isolated, writable
|
||||
Beyond linear history, LanceDB also supports **branches**: isolated, writable
|
||||
lines of history forked from `main` (or a specific version). Whereas tags and
|
||||
`checkout` give you read-only views of existing versions, a branch has its own
|
||||
writable history, making it ideal for experiments, backfills, and migrations
|
||||
|
||||
@@ -823,8 +823,8 @@ const shouldResults = await table
|
||||
<Info>
|
||||
**How to use booleans?**
|
||||
|
||||
- Use `&` (Python), `Occur.Must`(Typescript) for intersection (documents must match all queries).
|
||||
- Use `|` (Python), `Occur.Should`(Typescript) for union (documents must match at least one query).
|
||||
- Use `&` (Python), `Occur.Must` (TypeScript) for intersection (documents must match all queries).
|
||||
- Use `|` (Python), `Occur.Should` (TypeScript) for union (documents must match at least one query).
|
||||
</Info>
|
||||
|
||||
|
||||
|
||||
@@ -96,7 +96,7 @@ JSON scalar index. See [Functions](/sql/scalar-functions#json).
|
||||
|
||||
### Blobs {#blobs}
|
||||
|
||||
A blob column is read either as its binary contents or as a **description** — a
|
||||
A blob column is read either as its binary contents or as a **description**: a
|
||||
pointer plus metadata, without fetching the bytes. That choice changes the
|
||||
column's Arrow type, and therefore the schema your query sees.
|
||||
|
||||
@@ -121,7 +121,7 @@ error naming the missing feature rather than an unknown-function error, and the
|
||||
spatial index is gated the same way.
|
||||
</Warning>
|
||||
|
||||
Geometry can still be stored -- it is binary, and binary columns work -- but
|
||||
Geometry can still be stored (it is binary, and binary columns work), but
|
||||
nothing interprets it. Filter on a bounding box held in ordinary numeric
|
||||
columns if you need spatial predicates today.
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ Parameter and result types are Arrow type names, not SQL ones: `int64`,
|
||||
unless it is declared `NOT NULL`.
|
||||
|
||||
The body is Python source. Its dependencies are declared with it and are part
|
||||
of the **immutable function version** -- changing one produces a new version
|
||||
of the **immutable function version**. Changing one produces a new version
|
||||
rather than altering the registered one, so a column filled by an earlier
|
||||
version is unaffected.
|
||||
|
||||
@@ -40,7 +40,7 @@ version is unaffected.
|
||||
| | |
|
||||
| --- | --- |
|
||||
| `LANGUAGE` other than `python` | refused |
|
||||
| `pip` and `conda` together | refused -- pick one |
|
||||
| `pip` and `conda` together | refused (pick one) |
|
||||
| `conda_channels` without `conda` | refused |
|
||||
| `gpu` set to anything but `'true'` | refused |
|
||||
| An empty body | refused |
|
||||
@@ -69,13 +69,13 @@ DROP FUNCTION IF EXISTS embed WITH (version = '1')
|
||||
DROP FUNCTION embed WITH (version = '1', cascade = 'true')
|
||||
```
|
||||
|
||||
`version` is required -- a function is a set of immutable versions, and
|
||||
`version` is required: a function is a set of immutable versions, and
|
||||
dropping is always of one specific version, never "the function" as a whole.
|
||||
|
||||
<Warning>
|
||||
Without `cascade = 'true'`, dropping a version that a column still pins fails
|
||||
rather than breaking that column silently. Passing it drops the version and
|
||||
lets columns pinned to it stop refreshing -- they keep their last-filled
|
||||
lets columns pinned to it stop refreshing. They keep their last-filled
|
||||
values, per [Guarantees](/basics/guarantees#jobs).
|
||||
</Warning>
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
title: "Index"
|
||||
sidebarTitle: "Index"
|
||||
description: "CREATE INDEX and DROP INDEX — one column, with the algorithm chosen by USING."
|
||||
description: "CREATE INDEX and DROP INDEX: one column, with the algorithm chosen by USING."
|
||||
icon: bolt
|
||||
keywords: ["create index", "drop index", "sql"]
|
||||
---
|
||||
|
||||
@@ -8,8 +8,8 @@ keywords: ["show jobs", "show errors", "pause job", "resume job", "cancel job",
|
||||
|
||||
<Badge color="red">Enterprise</Badge> [SQL runs in LanceDB Enterprise.](/sql)
|
||||
|
||||
A [job](/basics/jobs) is not a catalog object -- it has no schema and is not
|
||||
namespaced -- but it has its own SQL statements, grouped here the same way the
|
||||
A [job](/basics/jobs) is not a catalog object (it has no schema and is not
|
||||
namespaced), but it has its own SQL statements, grouped here the same way the
|
||||
catalog objects are.
|
||||
|
||||
## SHOW JOBS {#show}
|
||||
@@ -18,7 +18,7 @@ catalog objects are.
|
||||
SHOW JOBS
|
||||
```
|
||||
|
||||
Every job the server knows about -- id, type, state, creation time. The SQL
|
||||
Every job the server knows about: id, type, state, creation time. The SQL
|
||||
counterpart of `list_jobs`.
|
||||
|
||||
## SHOW ERRORS {#errors}
|
||||
@@ -44,7 +44,7 @@ fewer filled rows than you expected.
|
||||
| Tables the server does not open directly | not supported |
|
||||
|
||||
Without `FOR JOB` this surveys recent failures and stops at 32 jobs. When the
|
||||
listing stops at your `LIMIT`, the message says so -- narrow with `FOR JOB`
|
||||
listing stops at your `LIMIT`, the message says so. Narrow with `FOR JOB`
|
||||
rather than raising the limit blindly.
|
||||
|
||||
## PAUSE JOB {#pause}
|
||||
@@ -66,7 +66,7 @@ CANCEL JOB 'j-ci-abc-def'
|
||||
```
|
||||
|
||||
The job id is always a quoted string literal. All three take exactly one and
|
||||
nothing after it -- a second token on the line is a syntax error rather than
|
||||
nothing after it: a second token on the line is a syntax error rather than
|
||||
being ignored.
|
||||
|
||||
<Note>
|
||||
|
||||
@@ -8,8 +8,8 @@ keywords: ["materialized view", "create materialized view", "show materialized v
|
||||
|
||||
<Badge color="red">Enterprise</Badge> [SQL runs in LanceDB Enterprise.](/sql)
|
||||
|
||||
A materialized view stores a query's result as rows, kept current by a refresh
|
||||
-- the counterpart to a [view](/sql/ddl/view), which stores only the query.
|
||||
A materialized view stores a query's result as rows, kept current by a refresh.
|
||||
It is the counterpart to a [view](/sql/ddl/view), which stores only the query.
|
||||
|
||||
<Note>
|
||||
Materialized views are off by default in LanceDB Enterprise and have to be enabled
|
||||
@@ -51,7 +51,7 @@ and compatibility notes for older server versions.
|
||||
## Dropping one {#drop}
|
||||
|
||||
A materialized view **is** a table, so it is dropped with
|
||||
[`DROP TABLE`](/sql/ddl/table#drop) -- which is also why dropping one removes
|
||||
[`DROP TABLE`](/sql/ddl/table#drop). For the same reason, dropping one removes
|
||||
stored rows, unlike dropping a [view](/sql/ddl/view#drop).
|
||||
|
||||
<Note>
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
---
|
||||
title: "Namespace"
|
||||
sidebarTitle: "Namespace"
|
||||
description: "CREATE SCHEMA and DROP SCHEMA — the SQL spelling of namespace DDL."
|
||||
description: "CREATE SCHEMA and DROP SCHEMA: the SQL spelling of namespace DDL."
|
||||
icon: folder-tree
|
||||
keywords: ["namespace", "schema", "create schema", "drop schema", "sql"]
|
||||
---
|
||||
|
||||
<Badge color="red">Enterprise</Badge> [SQL runs in LanceDB Enterprise.](/sql)
|
||||
|
||||
A [namespace](/management/namespace) is addressed in SQL as a `SCHEMA` -- the
|
||||
A [namespace](/management/namespace) is addressed in SQL as a `SCHEMA`, the
|
||||
standard's name for the same concept.
|
||||
|
||||
## CREATE SCHEMA {#create}
|
||||
@@ -39,8 +39,8 @@ DROP SCHEMA `a$b` RESTRICT
|
||||
<Warning>
|
||||
`CASCADE` is rejected: **"DROP SCHEMA CASCADE is not supported; drop the
|
||||
objects inside first."** A namespace's tables, views and other objects must be
|
||||
dropped explicitly before the namespace itself can go. `RESTRICT` -- the
|
||||
default -- states that requirement rather than changing it.
|
||||
dropped explicitly before the namespace itself can go. `RESTRICT` (the
|
||||
default) states that requirement rather than changing it.
|
||||
</Warning>
|
||||
|
||||
## What to read next {#next}
|
||||
|
||||
@@ -15,7 +15,7 @@ CREATE TABLE documents (id BIGINT, title VARCHAR, created_at TIMESTAMP)
|
||||
CREATE TABLE recent AS SELECT * FROM documents WHERE created_at > now() - INTERVAL '7 days'
|
||||
```
|
||||
|
||||
Column types are Arrow types under SQL names -- see [Data types](/sql/data-types).
|
||||
Column types are Arrow types under SQL names. See [Data types](/sql/data-types).
|
||||
A table inside a [namespace](/sql/ddl/namespace) is written as a dotted path:
|
||||
`prod.events.clicks`.
|
||||
|
||||
@@ -25,7 +25,7 @@ A table inside a [namespace](/sql/ddl/namespace) is written as a dotted path:
|
||||
DESCRIBE documents
|
||||
```
|
||||
|
||||
The table's columns and types, without a plan -- the SQL counterpart of
|
||||
The table's columns and types, without a plan. The SQL counterpart of
|
||||
`table.schema`.
|
||||
|
||||
## ALTER TABLE {#alter}
|
||||
@@ -37,7 +37,7 @@ name.
|
||||
|
||||
SQL `ADD COLUMN` requires an `AS` binding, so a plain column such as
|
||||
`ALTER TABLE t ADD COLUMN c INT` is rejected. Add an empty, nullable column
|
||||
with the SDK instead -- see
|
||||
with the SDK instead. See
|
||||
[Add nullable columns](/management/table#schema-add-nullable-columns).
|
||||
|
||||
### A computed column {#computed-column}
|
||||
@@ -47,7 +47,7 @@ ALTER TABLE t ADD COLUMN doubled AS value * 2
|
||||
```
|
||||
|
||||
`AS <expression>` declares a [computed column](/feature-engineering/add-computed-column).
|
||||
The column is committed empty and the expression is stored, not evaluated --
|
||||
The column is committed empty and the expression is stored, not evaluated:
|
||||
declaring one on a billion-row table costs what declaring one on an empty table
|
||||
costs. [`REFRESH COLUMN`](/sql/dml/refresh) fills it.
|
||||
|
||||
@@ -129,9 +129,9 @@ rows a sample would draw from are not all in the fragments yet.
|
||||
| --- | --- |
|
||||
| `ALTER TABLE ... DROP COLUMN` | not supported |
|
||||
|
||||
Dropping, renaming, or retyping a column is an SDK operation --
|
||||
[`drop_columns`](/management/table#schema) and
|
||||
[`alter_columns`](/management/table#schema) -- not a SQL statement.
|
||||
Dropping, renaming, or retyping a column is an SDK operation
|
||||
([`drop_columns`](/management/table#schema) and
|
||||
[`alter_columns`](/management/table#schema)), not a SQL statement.
|
||||
|
||||
## CLONE {#clone}
|
||||
|
||||
@@ -147,7 +147,7 @@ copying them, so it costs one metadata commit regardless of the source's size.
|
||||
|
||||
`AT` pins the point in the source's history to clone. A bare integer is a
|
||||
version and a bare quoted string is a timestamp; `VERSION`, `TAG` and
|
||||
`TIMESTAMP` say so explicitly, and a tag always needs its keyword -- a bare
|
||||
`TIMESTAMP` say so explicitly, and a tag always needs its keyword: a bare
|
||||
quoted string cannot be both a tag name and a timestamp.
|
||||
|
||||
<Warning>
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
title: "View"
|
||||
sidebarTitle: "View"
|
||||
description: "CREATE VIEW and DROP VIEW — a saved query, evaluated when read."
|
||||
description: "CREATE VIEW and DROP VIEW: a saved query, evaluated when read."
|
||||
icon: eye
|
||||
keywords: ["create view", "drop view", "sql", "catalog"]
|
||||
---
|
||||
@@ -9,7 +9,7 @@ keywords: ["create view", "drop view", "sql", "catalog"]
|
||||
<Badge color="red">Enterprise</Badge> [SQL runs in LanceDB Enterprise.](/sql)
|
||||
|
||||
A view stores a **query**, not rows. Reading it runs the query against current
|
||||
data, so a view is never stale -- and never faster than the query it wraps. It
|
||||
data, so a view is never stale, and never faster than the query it wraps. It
|
||||
is the counterpart to a [materialized view](/sql/ddl/materialized-view), which
|
||||
stores rows and is brought up to date by a refresh; both are
|
||||
[catalog objects](/basics/catalog) defined by a query, differing only in when
|
||||
@@ -27,8 +27,8 @@ FROM "documents" AS t1
|
||||
INNER JOIN "categories" AS t2 ON t1.ints = t2.ints
|
||||
```
|
||||
|
||||
The statement returns the view's schema -- column name, data type, nullability
|
||||
-- resolved from the query at creation time. Anything the
|
||||
The statement returns the view's schema (column name, data type, nullability),
|
||||
resolved from the query at creation time. Anything the
|
||||
[dialect](/sql/functions) supports can define one: joins, predicates,
|
||||
aggregates, window functions.
|
||||
|
||||
@@ -50,7 +50,7 @@ the view would have produced and then discarded.
|
||||
DROP VIEW "recent_matches"
|
||||
```
|
||||
|
||||
Removes the saved query. Unlike dropping a table, no data is removed -- there
|
||||
Removes the saved query. Unlike dropping a table, no data is removed: there
|
||||
was none to begin with.
|
||||
|
||||
## Listing {#listing}
|
||||
@@ -62,10 +62,10 @@ catalog distinguishes them:
|
||||
| --- | --- |
|
||||
| view names | the views in the database |
|
||||
| table names | the tables |
|
||||
| relation names | everything queryable -- both |
|
||||
| relation names | everything queryable (both) |
|
||||
|
||||
Use the relation listing when the question is "what can I select from," and the
|
||||
separate listings when the distinction matters -- before dropping, for
|
||||
separate listings when the distinction matters: before dropping, for
|
||||
instance, since dropping a view removes only the saved query while dropping a
|
||||
table removes data.
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
title: "Copy"
|
||||
sidebarTitle: "Copy"
|
||||
description: "COPY ... TO — writing a query result to object storage as files."
|
||||
description: "COPY ... TO: writing a query result to object storage as files."
|
||||
icon: file-export
|
||||
keywords: ["copy", "export", "parquet", "sql"]
|
||||
---
|
||||
@@ -17,7 +17,7 @@ COPY (SELECT id, title FROM documents) TO 's3://bucket/export/' STORED AS PARQUE
|
||||
```
|
||||
|
||||
Writes a query result to object storage. This is the export path when the
|
||||
consumer is another engine rather than an application -- results land as files
|
||||
consumer is another engine rather than an application: results land as files
|
||||
rather than streaming to a client.
|
||||
|
||||
## What to read next {#next}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
title: "Delete"
|
||||
sidebarTitle: "Delete"
|
||||
description: "DELETE FROM ... WHERE — rows are marked, not erased."
|
||||
description: "DELETE FROM ... WHERE: rows are marked, not erased."
|
||||
icon: trash
|
||||
keywords: ["delete", "sql"]
|
||||
---
|
||||
@@ -21,7 +21,7 @@ for as long as the version holding it survives
|
||||
[retention](/maintenance/cleanup).
|
||||
|
||||
Commits a new table version, so [versioning](/management/table#versioning) and
|
||||
the [write guarantees](/basics/guarantees#writes) apply -- including that the
|
||||
the [write guarantees](/basics/guarantees#writes) apply, including that the
|
||||
version advances even when the statement affects zero rows.
|
||||
|
||||
## What to read next {#next}
|
||||
|
||||
@@ -18,7 +18,7 @@ INSERT INTO archive SELECT * FROM documents WHERE created_at < '2026-01-01'
|
||||
server-side, without a round trip through your process.
|
||||
|
||||
Commits a new table version, so [versioning](/management/table#versioning) and
|
||||
the [write guarantees](/basics/guarantees#writes) apply -- including that the
|
||||
the [write guarantees](/basics/guarantees#writes) apply, including that the
|
||||
version advances even when the statement affects zero rows.
|
||||
|
||||
## What to read next {#next}
|
||||
@@ -28,6 +28,6 @@ version advances even when the statement affects zero rows.
|
||||
The SDK equivalent.
|
||||
</Card>
|
||||
<Card title="Merge Insert" icon="code-merge" href="/ingestion/merge-insert">
|
||||
Upsert -- matching incoming rows against existing keys -- has no SQL form.
|
||||
Upsert (matching incoming rows against existing keys) has no SQL form.
|
||||
</Card>
|
||||
</CardGroup>
|
||||
|
||||
@@ -27,7 +27,7 @@ REFRESH COLUMN embedding ON documents WHERE created_at > '2026-01-01'
|
||||
|
||||
A refresh gives values to rows that lack them, recomputes rows whose inputs
|
||||
changed, and **leaves other filled rows alone**. That makes it idempotent, cheap
|
||||
to repeat as data arrives, and safe to re-run after a failure -- though a re-run
|
||||
to repeat as data arrives, and safe to re-run after a failure, though a re-run
|
||||
can recompute values the failed run had already written.
|
||||
|
||||
`WHERE` scopes the refresh, which is how you fill a slice and check it before
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
title: "Update"
|
||||
sidebarTitle: "Update"
|
||||
description: "UPDATE ... SET ... WHERE — and why it does not recompute derived columns."
|
||||
description: "UPDATE ... SET ... WHERE, and why it does not recompute derived columns."
|
||||
icon: pen
|
||||
keywords: ["update", "sql"]
|
||||
---
|
||||
@@ -19,12 +19,12 @@ clause every row is updated.
|
||||
<Warning>
|
||||
An update does not recompute [computed columns](/feature-engineering/add-computed-column)
|
||||
that derive from the column you changed. Those rows keep the value computed from
|
||||
the old input until the next refresh recomputes them -- see
|
||||
the old input until the next refresh recomputes them. See
|
||||
[Guarantees](/basics/guarantees#jobs).
|
||||
</Warning>
|
||||
|
||||
Commits a new table version, so [versioning](/management/table#versioning) and
|
||||
the [write guarantees](/basics/guarantees#writes) apply -- including that the
|
||||
the [write guarantees](/basics/guarantees#writes) apply, including that the
|
||||
version advances even when the statement affects zero rows.
|
||||
|
||||
## What to read next {#next}
|
||||
|
||||
@@ -32,7 +32,7 @@ SELECT category, source, count(*) FROM documents GROUP BY category, source
|
||||
|
||||
## Aggregate without grouping {#whole-table}
|
||||
|
||||
Omitting `GROUP BY` aggregates the whole table into one row -- the usual way to
|
||||
Omitting `GROUP BY` aggregates the whole table into one row, which is the usual way to
|
||||
ask how big something is:
|
||||
|
||||
```sql
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
title: "Explain"
|
||||
sidebarTitle: "Explain"
|
||||
description: "EXPLAIN, EXPLAIN ANALYZE and EXPLAIN VERBOSE — reading and measuring a plan from SQL."
|
||||
description: "EXPLAIN, EXPLAIN ANALYZE and EXPLAIN VERBOSE: reading and measuring a plan from SQL."
|
||||
icon: map
|
||||
keywords: ["explain", "explain analyze", "explain verbose", "query plan"]
|
||||
---
|
||||
@@ -20,7 +20,7 @@ EXPLAIN ANALYZE SELECT * FROM documents WHERE identifier < 1000
|
||||
| `EXPLAIN VERBOSE <query>` | Adds intermediate optimizer plans on top of `EXPLAIN`. |
|
||||
|
||||
The result is a small relation with `plan_type` and `plan` columns, read
|
||||
through the same client you use for regular queries -- see
|
||||
through the same client you use for regular queries. See
|
||||
[Connecting](/sql#connecting).
|
||||
|
||||
SQL plans and the SDK's
|
||||
@@ -46,7 +46,7 @@ SQL can also add joins, aggregates and other operators:
|
||||
| `FilterExec` above a search | the filter ran after search rather than pushing into it |
|
||||
| `ANNSubIndex` / `ANNIvfPartition` | the vector index was used |
|
||||
|
||||
A `LanceScan` feeding a join is the common one worth acting on -- see
|
||||
A `LanceScan` feeding a join is the common one worth acting on. See
|
||||
[Joins](/sql/dql/join#performance).
|
||||
|
||||
For a table's columns and types without a plan, see
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
title: "Full-Text Search"
|
||||
sidebarTitle: "Full-Text Search"
|
||||
description: "The fts() table function — match, fuzzy, phrase, boolean, boost and multi-match queries from SQL."
|
||||
description: "The fts() table function: match, fuzzy, phrase, boolean, boost and multi-match queries from SQL."
|
||||
icon: magnifying-glass
|
||||
keywords: ["fts", "sql", "full text search", "fuzzy", "phrase", "boolean", "boost", "multi-match"]
|
||||
---
|
||||
@@ -9,7 +9,7 @@ keywords: ["fts", "sql", "full text search", "fuzzy", "phrase", "boolean", "boos
|
||||
<Badge color="red">Enterprise</Badge> [SQL runs in LanceDB Enterprise.](/sql)
|
||||
|
||||
Full-text search is a **table function**, `fts(table, query)`, rather than a
|
||||
predicate -- it produces a relevance score alongside the matching rows, and a
|
||||
predicate: it produces a relevance score alongside the matching rows, and a
|
||||
`WHERE` clause has nowhere to put one. The two string-literal arguments are
|
||||
the table name and a JSON-encoded query, built with the same query-builder
|
||||
classes used from the SDK and serialized with `.to_json()`.
|
||||
@@ -32,8 +32,8 @@ CREATE INDEX ON my_docs USING fts (text);
|
||||
```
|
||||
|
||||
<Warning>
|
||||
Phrase queries need position information, which costs extra index size --
|
||||
add `WITH (with_position = true)` to `CREATE INDEX`. Without it,
|
||||
Phrase queries need position information, which costs extra index size.
|
||||
Add `WITH (with_position = true)` to `CREATE INDEX`. Without it,
|
||||
[phrase queries](#phrase) find nothing. See
|
||||
[FTS parameters](/indexing/fts-index#fts-parameters).
|
||||
</Warning>
|
||||
@@ -57,7 +57,7 @@ FROM fts('my_docs', '{"match":{"column":"text","terms":"puppy"}}')
|
||||
LIMIT 5
|
||||
```
|
||||
|
||||
By default the top 5 matches come back in **arbitrary order** -- select
|
||||
By default the top 5 matches come back in **arbitrary order**. Select
|
||||
`_score` and order by it for relevance ranking:
|
||||
|
||||
```sql
|
||||
@@ -126,8 +126,8 @@ MultiMatchQuery("puppy", ["text", "category"], boosts=[2.0, 1.0]) # text weight
|
||||
|
||||
## Combining with SQL {#combining}
|
||||
|
||||
`fts()` is a relation, so it composes with the rest of the SQL surface --
|
||||
filter, join, or aggregate over it same as any table:
|
||||
`fts()` is a relation, so it composes with the rest of the SQL surface.
|
||||
Filter, join, or aggregate over it same as any table:
|
||||
|
||||
```sql
|
||||
SELECT id, text, category
|
||||
|
||||
@@ -24,14 +24,14 @@ JOIN categories AS c ON c.id = d.category_id
|
||||
| `LEFT [OUTER] JOIN` | every left row, nulls where the right has no match |
|
||||
| `RIGHT [OUTER] JOIN` | every right row |
|
||||
| `FULL [OUTER] JOIN` | every row from both |
|
||||
| `CROSS JOIN` | every combination -- the product of the two |
|
||||
| `CROSS JOIN` | every combination (the product of the two) |
|
||||
|
||||
`USING (col)` is shorthand for an equality join on a shared column name, and
|
||||
avoids repeating it in the select list.
|
||||
|
||||
## Across namespaces {#namespaces}
|
||||
|
||||
Tables in different [namespaces](/management/namespace) join by their dotted paths -- the
|
||||
Tables in different [namespaces](/management/namespace) join by their dotted paths. The
|
||||
catalog is one catalog, so there is nothing to federate:
|
||||
|
||||
```sql
|
||||
@@ -56,7 +56,7 @@ ORDER BY hits DESC
|
||||
```
|
||||
|
||||
That composition is the reason full-text search is a relation rather than a
|
||||
predicate -- a `WHERE` clause has nowhere to put the relevance score.
|
||||
predicate: a `WHERE` clause has nowhere to put the relevance score.
|
||||
|
||||
## Performance {#performance}
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ SELECT * FROM documents TABLESAMPLE SYSTEM (10)
|
||||
| `SYSTEM` | `BLOCK` | **Lance fragments**, and takes whatever rows they hold |
|
||||
|
||||
`BERNOULLI` gives a genuinely uniform sample and reads across the whole table to
|
||||
get it. `SYSTEM` reads far less -- it skips entire fragments -- but the sample is
|
||||
get it. `SYSTEM` reads far less (it skips entire fragments), but the sample is
|
||||
only as uniform as the data's layout. If rows were written in an order that
|
||||
correlates with what you are measuring, a fragment sample inherits that
|
||||
correlation.
|
||||
@@ -46,7 +46,7 @@ independent row sampling. Omitting the method gives
|
||||
|
||||
## Repeatability {#repeatable}
|
||||
|
||||
Without a seed, every execution resamples -- running the same query twice gives
|
||||
Without a seed, every execution resamples: running the same query twice gives
|
||||
different rows. `REPEATABLE` fixes the draw:
|
||||
|
||||
```sql
|
||||
@@ -98,5 +98,5 @@ would draw from are not all in the fragments yet.
|
||||
## Sampling outside SQL {#sdk}
|
||||
|
||||
For a training loop rather than a query, the SDKs read a table in a shuffled
|
||||
order without going through SQL at all -- see [Training](/training/data-loading), where a
|
||||
order without going through SQL at all. See [Training](/training/data-loading), where a
|
||||
permutation drives the data loader.
|
||||
|
||||
@@ -107,8 +107,8 @@ A CTE is scoped to the statement. One you want to keep is a
|
||||
|
||||
## Subqueries {#subqueries}
|
||||
|
||||
Subqueries are allowed anywhere an expression or a relation is -- in `FROM`, in
|
||||
`WHERE` with `IN` or `EXISTS`, and as a scalar in the select list:
|
||||
Subqueries are allowed anywhere an expression or a relation is (in `FROM`, in
|
||||
`WHERE` with `IN` or `EXISTS`, and as a scalar in the select list):
|
||||
|
||||
```sql
|
||||
SELECT title
|
||||
|
||||
@@ -89,7 +89,7 @@ WITH ranked AS (
|
||||
SELECT * FROM ranked WHERE rn = 1
|
||||
```
|
||||
|
||||
The outer query is needed because a window function cannot appear in `WHERE` --
|
||||
The outer query is needed because a window function cannot appear in `WHERE`:
|
||||
windows are computed after filtering, so the predicate would have nothing to
|
||||
read.
|
||||
|
||||
|
||||
@@ -14,9 +14,9 @@ plus what Lance adds.
|
||||
|
||||
| | |
|
||||
| --- | --- |
|
||||
| [Scalar functions](/sql/scalar-functions) | per row — math, string, regex, date/time, array, struct, JSON |
|
||||
| [Aggregate functions](/sql/dql/aggregation#functions) | per group — `count`, `sum`, statistical, approximate |
|
||||
| [Window functions](/sql/dql/window#functions) | per window — ranking, offset, running aggregates |
|
||||
| [Scalar functions](/sql/scalar-functions) | per row: math, string, regex, date/time, array, struct, JSON |
|
||||
| [Aggregate functions](/sql/dql/aggregation#functions) | per group: `count`, `sum`, statistical, approximate |
|
||||
| [Window functions](/sql/dql/window#functions) | per window: ranking, offset, running aggregates |
|
||||
| [Table functions](#table-functions) | produce a relation rather than a value |
|
||||
|
||||
<Note>
|
||||
@@ -51,7 +51,7 @@ and multi-match queries use this JSON encoding.
|
||||
|
||||
Its schema contains the dataset's columns plus `_score`, so every stored
|
||||
column is available in the result. Projections push into the scan.
|
||||
Being a relation is also what makes it compose — join it, filter it, aggregate
|
||||
Being a relation is also what makes it compose: join it, filter it, aggregate
|
||||
over it. This join additionally requires `my_docs.category_id` and a
|
||||
`categories` table with `id` and `name` columns:
|
||||
|
||||
@@ -73,7 +73,7 @@ multi-match query forms.
|
||||
### `_score` {#score}
|
||||
|
||||
A pseudo-column produced by `fts()`, carrying relevance for the row. It behaves
|
||||
like any other column — order by it, filter on it, project it — and exists only
|
||||
like any other column (order by it, filter on it, project it) and exists only
|
||||
in a relation that came from `fts()`. See
|
||||
[Data types](/sql/data-types#pseudo-columns).
|
||||
|
||||
|
||||
@@ -38,7 +38,7 @@ collecting the entire result in application memory. See
|
||||
|
||||
The endpoint executes one statement per request and is meant for the queries
|
||||
and DDL/DML pages below. Reach for the SDKs instead for table-management
|
||||
operations that have no SQL form, such as a merge insert -- see
|
||||
operations that have no SQL form, such as a merge insert. See
|
||||
[What SQL cannot do](#cannot).
|
||||
|
||||
## Reference {#reference}
|
||||
@@ -66,7 +66,7 @@ Draft for the Functions release. Add this row to the table above, after Index:
|
||||
| [Insert](/sql/dml/insert) | `INSERT INTO`, including `INSERT ... SELECT` |
|
||||
| [Update](/sql/dml/update) | `UPDATE ... SET` |
|
||||
| [Delete](/sql/dml/delete) | `DELETE FROM` |
|
||||
| [Copy](/sql/dml/copy) | `COPY` -- refused by recent deployment versions |
|
||||
| [Copy](/sql/dml/copy) | `COPY` (refused by recent deployment versions) |
|
||||
| [Refresh](/sql/dml/refresh) | filling computed columns and materialized views |
|
||||
|
||||
### DQL {#dql}
|
||||
@@ -97,9 +97,9 @@ Draft for the Functions release. Add this row to the table above, after Index:
|
||||
| | |
|
||||
| --- | --- |
|
||||
| Vector search | See [SQL vector search](/basics/sql#no-vector-sql) for supported query forms and server-version requirements. The [SDK query builder](/search/vector-search) also supports vector search. |
|
||||
| Merge insert | no SQL form -- needs a source relation and a per-match policy |
|
||||
| Merge insert | no SQL form: it needs a source relation and a per-match policy |
|
||||
| `information_schema` | not enabled, so `SHOW TABLES` and `SHOW COLUMNS` fail |
|
||||
| `COPY` | refused by recent deployment versions -- see [Copy](/sql/dml/copy) |
|
||||
| `COPY` | refused by recent deployment versions (see [Copy](/sql/dml/copy)) |
|
||||
| `PREPARE` / `EXECUTE` | preparation is a [protocol feature](/sql/prepared-statements), not a statement |
|
||||
| Run on a local database | no engine to plan the statement |
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ operator precedence.
|
||||
| | |
|
||||
| --- | --- |
|
||||
| `=` `!=` `<>` `<` `<=` `>` `>=` | ordinary comparison |
|
||||
| `IS NULL` / `IS NOT NULL` | null test — `= NULL` is never true |
|
||||
| `IS NULL` / `IS NOT NULL` | null test (`= NULL` is never true) |
|
||||
| `IS DISTINCT FROM` | comparison that treats null as a value |
|
||||
| `BETWEEN a AND b` | inclusive range |
|
||||
| `IN (...)` | set membership |
|
||||
@@ -34,7 +34,7 @@ its wildcard semantics.
|
||||
|
||||
`AND`, `OR`, `NOT`. Three-valued: a comparison against null is **unknown**, and
|
||||
`NOT unknown` is still unknown. A `WHERE` clause keeps only rows where the
|
||||
predicate is true, so unknown behaves like false there — but not inside `NOT
|
||||
predicate is true, so unknown behaves like false there, but not inside `NOT
|
||||
IN`, which is the usual place this bites. See
|
||||
[Subqueries](/sql/dql/subqueries#in-exists).
|
||||
|
||||
@@ -43,7 +43,7 @@ IN`, which is the usual place this bites. See
|
||||
`+` `-` `*` `/` `%`, with the usual precedence. String concatenation is `||`.
|
||||
|
||||
<Warning>
|
||||
Arithmetic over a vector column is **list** arithmetic, not scalar arithmetic —
|
||||
Arithmetic over a vector column is **list** arithmetic, not scalar arithmetic:
|
||||
a vector is a `FixedSizeList<Float32>`. When the operation you want is a
|
||||
similarity comparison, use [vector search](/search/vector-search); whether SQL
|
||||
can express it [depends on the Enterprise server version](/basics/sql#no-vector-sql).
|
||||
@@ -53,7 +53,7 @@ can express it [depends on the Enterprise server version](/basics/sql#no-vector-
|
||||
|
||||
| | |
|
||||
| --- | --- |
|
||||
| String | `'text'` — doubled quote to escape: `'it''s'` |
|
||||
| String | `'text'`, with a doubled quote to escape: `'it''s'` |
|
||||
| Number | `42`, `3.14` |
|
||||
| Boolean | `TRUE`, `FALSE` |
|
||||
| Null | `NULL` |
|
||||
@@ -61,8 +61,8 @@ can express it [depends on the Enterprise server version](/basics/sql#no-vector-
|
||||
| Interval | `INTERVAL '7 days'` |
|
||||
| Array | `[1, 2, 3]` |
|
||||
|
||||
Identifiers are quoted with double quotes when they need to be — a reserved word
|
||||
or a name with mixed case: `SELECT "order" FROM "MyTable"`.
|
||||
Identifiers are quoted with double quotes when they need to be (a reserved word
|
||||
or a name with mixed case): `SELECT "order" FROM "MyTable"`.
|
||||
|
||||
## Casting {#casting}
|
||||
|
||||
|
||||
@@ -143,7 +143,7 @@ punctuation. Equivalent to an `fts()` match query with fuzziness off, `AND`
|
||||
semantics and the simple tokenizer.
|
||||
|
||||
**Index-accelerated.** When the column has a full-text index that suits the
|
||||
query, the planner uses it instead of scanning — which is the reason to prefer
|
||||
query, the planner uses it instead of scanning, which is the reason to prefer
|
||||
it over `LIKE` for keyword tests.
|
||||
|
||||
### JSON accessors {#json}
|
||||
|
||||
@@ -69,7 +69,7 @@ well as on LanceDB Enterprise (remote) tables.
|
||||
|
||||
## Connect to a table {#connect-to-a-table}
|
||||
|
||||
The branch API is identical no matter how you connect — only the connection
|
||||
The branch API is identical no matter how you connect: only the connection
|
||||
itself differs between OSS and Enterprise. Establish a connection (`db`) and open
|
||||
a `table` (see [Ingesting data](/ingestion)), then use the same branch calls
|
||||
in every example that follows.
|
||||
@@ -150,7 +150,7 @@ somewhere else, pass a branch name, a specific version, or both.
|
||||
|
||||
### Write to a branch {#write-to-a-branch}
|
||||
|
||||
Writes go through the branch handle and stay there — the `main` handle keeps
|
||||
Writes go through the branch handle and stay there: the `main` handle keeps
|
||||
reporting its original row count. Listing branches returns a mapping of each
|
||||
branch name to its metadata, including the version it was forked from.
|
||||
|
||||
@@ -170,7 +170,7 @@ branch name to its metadata, including the version it was forked from.
|
||||
|
||||
### Reopen a branch {#reopen-a-branch}
|
||||
|
||||
A branch outlives the handle that created it. Reopen it later by name — either
|
||||
A branch outlives the handle that created it. Reopen it later by name, either
|
||||
from an existing table handle or straight from the connection when you open the
|
||||
table. Both routes give you a writable handle tracking the branch's latest state.
|
||||
|
||||
@@ -193,7 +193,7 @@ table. Both routes give you a writable handle tracking the branch's latest state
|
||||
Deleting a branch removes it and its branch-local history; `main` is untouched.
|
||||
A deployment refuses to delete a branch whose columns have been promoted onto
|
||||
`main`, because `main` still depends on it.
|
||||
Before deleting a branch, make sure you've retained any results you need — see
|
||||
Before deleting a branch, make sure you've retained any results you need. See
|
||||
[Apply branch-tested changes to `main`](#apply-branch-tested-changes-to-main)
|
||||
below.
|
||||
|
||||
@@ -215,8 +215,8 @@ below.
|
||||
|
||||
<Warning>
|
||||
A branch has its own writable history. Outside of the [diff and cherry-pick
|
||||
APIs](#compare-and-merge-a-branch-into-main) — which are available on
|
||||
LanceDB Enterprise and promote added columns only — LanceDB does not reconcile
|
||||
APIs](#compare-and-merge-a-branch-into-main) (which are available on
|
||||
LanceDB Enterprise and promote added columns only), LanceDB does not reconcile
|
||||
one branch's history with another or detect conflicts between them. To carry
|
||||
other accepted work forward, rerun the validated operation against `main` or
|
||||
explicitly write selected results to it.
|
||||
@@ -242,7 +242,7 @@ If the result you want to retain is a set of inserted or updated rows, use
|
||||
matches incoming rows by key and does not merge branch histories.
|
||||
|
||||
Read the rows you want from the reviewed branch, then upsert them into `main` on
|
||||
your key column — `id` in this example. Here, `candidate` is the branch handle:
|
||||
your key column (`id` in this example). Here, `candidate` is the branch handle:
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="Python" language="Python" icon="python">
|
||||
@@ -259,7 +259,7 @@ your key column — `id` in this example. Here, `candidate` is the branch handle
|
||||
</CodeGroup>
|
||||
|
||||
With LanceDB Enterprise, the synchronous Python table raises for `to_arrow()`; read
|
||||
the branch's rows with a query instead -- see
|
||||
the branch's rows with a query instead. See
|
||||
[Avoid materializing the whole table](/search#avoid-materializing-the-whole-table).
|
||||
|
||||
This operation transfers only the inserted or updated rows in its input. It does
|
||||
@@ -293,8 +293,8 @@ row and index changes.
|
||||
|
||||
`diff` reads the branch and `main` without modifying either, and returns what
|
||||
has changed: columns added, removed or changed, indexes added or removed, row
|
||||
counts, and a list of `errors` -- the reasons a cherry-pick of the branch could
|
||||
not land now. An empty list means it could.
|
||||
counts, and a list of `errors` (the reasons a cherry-pick of the branch could
|
||||
not land now). An empty list means it could.
|
||||
|
||||
<CodeGroup>
|
||||
```python Python icon="python"
|
||||
@@ -436,6 +436,6 @@ give you to work with table history. Choose these approaches based on whether yo
|
||||
| **[`checkout(version)`](/management/table#versioning-rollback-to-previous-versions)** | ❌ No | Read a historical version of `main` without forking. Read-only until you `restore`. |
|
||||
|
||||
<Note>
|
||||
For linear version history — creating versions, listing them, rolling back, and
|
||||
tagging — see the [Versioning and Reproducibility](/management/table#versioning) guide.
|
||||
For linear version history (creating versions, listing them, rolling back, and
|
||||
tagging), see the [Versioning and Reproducibility](/management/table#versioning) guide.
|
||||
</Note>
|
||||
|
||||
@@ -155,7 +155,7 @@ for sample in eval_ds:
|
||||
|
||||
### Parallelism & Elasticity
|
||||
|
||||
When loading data, `StreamingDataset` partitions the the source dataset among disjoint parallel threads called **splits**. Each rank is allocated a contiguous group of splits, and each DataLoader worker associated to a rank gets a contiguous subgroup of its rank's splits.
|
||||
When loading data, `StreamingDataset` partitions the source dataset among disjoint parallel threads called **splits**. Each rank is allocated a contiguous group of splits, and each DataLoader worker associated to a rank gets a contiguous subgroup of its rank's splits.
|
||||
|
||||
If using `shuffle=False`, data is allocated to splits naively in ordered contiguous blocks of equal size. With `shuffle=True`
|
||||
(the default), the rows of the source dataset are shuffled and distributed among splits such that each split receives the same
|
||||
|
||||
@@ -5,7 +5,7 @@ description: End-to-end fine-tuning of a Faster R-CNN object detector on curated
|
||||
icon: car
|
||||
---
|
||||
|
||||
This example walks through fine-tuning an autonomous vehicle (AV) perception model on targeted failure-mode slices of [BDD100K](https://www.bdd100k.com/) — riders, nighttime pedestrians, and distant pedestrians — using LanceDB as a single multimodal table from raw JPEG bytes through to the PyTorch training loop.
|
||||
This example walks through fine-tuning an autonomous vehicle (AV) perception model on targeted failure-mode slices of [BDD100K](https://www.bdd100k.com/) (riders, nighttime pedestrians, and distant pedestrians) using LanceDB as a single multimodal table from raw JPEG bytes through to the PyTorch training loop.
|
||||
|
||||
The full pipeline lives in the [lancedb/training](https://github.com/lancedb/training/tree/main/object-detection) repository. This page focuses on the parts most relevant to training: defining curated splits as materialized views, loading them through the [`Permutation`](/training/data-loading) API, and pinning checkpoints to an exact data version.
|
||||
|
||||
@@ -22,13 +22,13 @@ Fine-tuning Faster R-CNN ResNet50 FPN v2 for 10 epochs on each curated slice (ba
|
||||
| **Distant pedestrian** | mAP@0.5 | 0.4746 | **0.5788** | **+22.0%** |
|
||||
| | Recall | 0.6794 | **0.8024** | **+18.1%** |
|
||||
|
||||
No external data added — only training-distribution correction via SQL filters over a single Lance table. Each panel below shows the same frame with three overlaid predictions: **green** = ground truth · **red** = pretrained COCO baseline · **blue** = fine-tuned model.
|
||||
No external data added, only training-distribution correction via SQL filters over a single Lance table. Each panel below shows the same frame with three overlaid predictions: **green** = ground truth · **red** = pretrained COCO baseline · **blue** = fine-tuned model.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
The rest of the page walks through the pipeline that produced these checkpoints.
|
||||
|
||||
@@ -42,7 +42,7 @@ A perception model fine-tuned on a generic dataset typically misses the long-tai
|
||||
| **Nighttime pedestrians** | `timeofday = 'night' AND has_person = true` |
|
||||
| **Distant pedestrians** | `has_person = true AND person_bbox_area_pct < 30.0` |
|
||||
|
||||
Each curated slice becomes a [Geneva materialized view](/geneva/jobs/materialized-views) — a named, refreshable SQL filter over the source table — and the training script loads it by name. New footage flows in through `add()` → `backfill()` → `refresh()`; no manifests, no exports, no reshuffling on disk.
|
||||
Each curated slice becomes a [Geneva materialized view](/geneva/jobs/materialized-views) (a named, refreshable SQL filter over the source table), and the training script loads it by name. New footage flows in through `add()` → `backfill()` → `refresh()`; no manifests, no exports, no reshuffling on disk.
|
||||
|
||||
## 1\. Schema {#1-schema}
|
||||
|
||||
@@ -70,7 +70,7 @@ BDD_SCHEMA = pa.schema([
|
||||
])
|
||||
```
|
||||
|
||||
Ingestion streams `pa.RecordBatch`es of raw frames + annotations directly into a Lance table — no intermediate preprocessing job. The table can live on local disk, S3, GCS, or Azure; everything downstream (backfills, views, the training loader) opens it in place via `lancedb.connect("s3://...")` with no local copy step.
|
||||
Ingestion streams `pa.RecordBatch`es of raw frames + annotations directly into a Lance table, with no intermediate preprocessing job. The table can live on local disk, S3, GCS, or Azure; everything downstream (backfills, views, the training loader) opens it in place via `lancedb.connect("s3://...")` with no local copy step.
|
||||
|
||||
## 2\. Backfill curation features with Geneva {#2-backfill-curation-features-with-geneva}
|
||||
|
||||
@@ -117,12 +117,12 @@ with gconn.local_ray_context():
|
||||
```
|
||||
|
||||
<Note>
|
||||
Because the curation features are flat scalar columns on the same table, all four retrieval modes — SQL, full-text search, vector search, and SQL-filtered vector search — work directly without joins or exports. See [Geneva backfilling](/geneva/jobs/backfilling) for how a backfill fills only the rows that lack values.
|
||||
Because the curation features are flat scalar columns on the same table, all four retrieval modes (SQL, full-text search, vector search, and SQL-filtered vector search) work directly without joins or exports. See [Geneva backfilling](/geneva/jobs/backfilling) for how a backfill fills only the rows that lack values.
|
||||
</Note>
|
||||
|
||||
## 3\. Define training splits as materialized views {#3-define-training-splits-as-materialized-views}
|
||||
|
||||
A training split is a named SQL filter, not a CSV manifest. Each view stays in sync with the source table and bumps its `version` on every refresh — the link between a checkpoint and the exact data that produced it.
|
||||
A training split is a named SQL filter, not a CSV manifest. Each view stays in sync with the source table and bumps its `version` on every refresh. That version is the link between a checkpoint and the exact data that produced it.
|
||||
|
||||
```py Python icon=Python
|
||||
import geneva
|
||||
@@ -155,7 +155,7 @@ with gconn.local_ray_context():
|
||||
|
||||
## 4\. PyTorch DataLoader via the Permutation API {#4-pytorch-dataloader-via-the-permutation-api}
|
||||
|
||||
The training script doesn't know about the filter — it opens a view by name and reads through the [`Permutation`](/training/data-loading) API. Each DataLoader worker reopens its own connection lazily, reads Arrow batches directly from Lance (zero-copy, no intermediate file format), and the collate function decodes the whole batch in one pass. `Permutation` provides random-access indexing over the table, so shuffling is a cheap pointer rewrite rather than a full-dataset shuffle on disk.
|
||||
The training script doesn't know about the filter: it opens a view by name and reads through the [`Permutation`](/training/data-loading) API. Each DataLoader worker reopens its own connection lazily, reads Arrow batches directly from Lance (zero-copy, no intermediate file format), and the collate function decodes the whole batch in one pass. `Permutation` provides random-access indexing over the table, so shuffling is a cheap pointer rewrite rather than a full-dataset shuffle on disk.
|
||||
|
||||
```py Python icon=Python
|
||||
import lancedb
|
||||
@@ -246,12 +246,12 @@ def make_loader(uri, table_name, batch_size=64, num_workers=8, shuffle=False):
|
||||
```
|
||||
|
||||
<Note>
|
||||
`with_format("arrow")` keeps batches as zero-copy `pa.RecordBatch`es — no per-row Python boxing, no pickling between worker and main. Each DataLoader worker reopens its own `Permutation` after fork (the Rust async handle is cleared in `__getstate__`), so reads scale with `num_workers` and stream straight from the underlying object store. JPEG decode overlaps with GPU compute via `pin_memory` + `prefetch_factor`, which is what keeps the loader from becoming the bottleneck on a fast GPU.
|
||||
`with_format("arrow")` keeps batches as zero-copy `pa.RecordBatch`es: no per-row Python boxing, no pickling between worker and main. Each DataLoader worker reopens its own `Permutation` after fork (the Rust async handle is cleared in `__getstate__`), so reads scale with `num_workers` and stream straight from the underlying object store. JPEG decode overlaps with GPU compute via `pin_memory` + `prefetch_factor`, which is what keeps the loader from becoming the bottleneck on a fast GPU.
|
||||
</Note>
|
||||
|
||||
## 5\. Fine-tune Faster R-CNN {#5-fine-tune-faster-r-cnn}
|
||||
|
||||
The training loop is plain PyTorch — the Lance integration ends at the loader. Mixed precision is enabled on CUDA for ~2× speedup on Ampere GPUs.
|
||||
The training loop is plain PyTorch: the Lance integration ends at the loader. Mixed precision is enabled on CUDA for ~2× speedup on Ampere GPUs.
|
||||
|
||||
```py Python icon=Python
|
||||
import time
|
||||
@@ -312,7 +312,7 @@ for epoch in range(1, 11):
|
||||
|
||||
## 6\. Pin the checkpoint to a data version {#6-pin-the-checkpoint-to-a-data-version}
|
||||
|
||||
Every Lance table — including a materialized view — exposes a monotonically increasing `version`. Logging it next to the weights gives a permanent, deterministic link between a checkpoint and the exact data snapshot that produced it.
|
||||
Every Lance table, including a materialized view, exposes a monotonically increasing `version`. Logging it next to the weights gives a permanent, deterministic link between a checkpoint and the exact data snapshot that produced it.
|
||||
|
||||
```py Python icon=Python
|
||||
import json
|
||||
@@ -340,7 +340,7 @@ tbl.checkout(version=7) # exact snapshot the checkpoint was trained on
|
||||
|
||||
## 7\. Continuous updates {#7-continuous-updates}
|
||||
|
||||
When new footage arrives, the same three calls update every downstream view — no view definitions change, no training-script edits required:
|
||||
When new footage arrives, the same three calls update every downstream view (no view definitions change, no training-script edits required):
|
||||
|
||||
```py Python icon=Python
|
||||
# 1. ingest the new footage into the source table
|
||||
@@ -361,7 +361,7 @@ for view_name in gconn.table_names():
|
||||
print(f"[{view_name}] {before} → {mv.count_rows()} rows (version {mv.version})")
|
||||
```
|
||||
|
||||
The next training run picks up the new data automatically — and pins itself to the new `version`.
|
||||
The next training run picks up the new data automatically and pins itself to the new `version`.
|
||||
|
||||
## Full source {#full-source}
|
||||
|
||||
|
||||
@@ -4,8 +4,8 @@ sidebarTitle: "Agentic RAG"
|
||||
description: "Several agents collaborating to retrieve and answer."
|
||||
---
|
||||
|
||||
Builds a RAG system where multiple agents collaborate -- retrieving
|
||||
information, checking it, and generating answers -- rather than a single
|
||||
Builds a RAG system where multiple agents collaborate (retrieving
|
||||
information, checking it, and generating answers) rather than a single
|
||||
retrieve-then-generate pass.
|
||||
|
||||
<Note>
|
||||
|
||||
@@ -6,7 +6,7 @@ description: "Prepend LLM-generated context to each chunk before embedding it."
|
||||
|
||||
Addresses the "lost in the middle" problem in retrieval. An LLM generates a
|
||||
succinct piece of context for each document chunk, which is prepended to the
|
||||
chunk before embedding -- so a chunk carries enough surrounding meaning to be
|
||||
chunk before embedding, so a chunk carries enough surrounding meaning to be
|
||||
retrieved for the right queries.
|
||||
|
||||
<Note>
|
||||
|
||||
@@ -4,8 +4,8 @@ sidebarTitle: "HyDE"
|
||||
description: "Generate a hypothetical answer, then retrieve documents similar to it."
|
||||
---
|
||||
|
||||
An LLM writes a hypothetical document answering the query, and that document --
|
||||
not the query -- is embedded and used for retrieval. It works because a
|
||||
An LLM writes a hypothetical document answering the query, and that document,
|
||||
not the query, is embedded and used for retrieval. It works because a
|
||||
plausible answer sits nearer the real answers in embedding space than a question
|
||||
does.
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ description: "Nested, variable-size embeddings with LanceDB and LlamaIndex."
|
||||
---
|
||||
|
||||
A RAG pipeline built on Matryoshka embeddings with LlamaIndex. The embeddings
|
||||
nest, so one model serves several dimensionalities -- store the full vector and
|
||||
nest, so one model serves several dimensionalities: store the full vector and
|
||||
retrieve with a truncated one when the cheaper comparison is good enough.
|
||||
|
||||
<Note>
|
||||
|
||||
@@ -150,7 +150,7 @@ This example is meant to prove more than a trivial vector lookup.
|
||||
- the adapter supports `RRFReranker`, `MRRReranker`, and `CrossEncoderReranker`
|
||||
- the default example uses `MRRReranker`, not a plain weighted linear combination
|
||||
|
||||
That matters for NVIDIA partner workloads because product names, storage platforms, and technical
|
||||
Hybrid retrieval is important for NVIDIA partner workloads because product names, storage platforms, and technical
|
||||
jargon often need exact lexical matching as well as semantic retrieval.
|
||||
|
||||
## How this can be extended {#how-this-can-be-extended}
|
||||
|
||||
@@ -11,7 +11,7 @@ All the scripts and code for this tutorial are available in the
|
||||
|
||||
## Use case: Financial services regulatory knowledge base {#use-case-financial-services-regulatory-knowledge-base}
|
||||
|
||||
Imagine you're a major investment bank. Your team is tasked with building a critical Retrieval-Augmented Generation (RAG) system. This system must provide instant, accurate answers to compliance officers about ever-changing financial regulations. A wrong or out-of-date answer isn't just an inconvenience—it could lead to multi-million dollar fines, reputational damage, and regulatory audits.
|
||||
Imagine you're a major investment bank. Your team is tasked with building a critical Retrieval-Augmented Generation (RAG) system. This system must provide instant, accurate answers to compliance officers about ever-changing financial regulations. A wrong or out-of-date answer isn't just an inconvenience: it could lead to multi-million dollar fines, reputational damage, and regulatory audits.
|
||||
|
||||
Your knowledge base is a living entity, constantly evolving with:
|
||||
|
||||
|
||||
@@ -53,7 +53,7 @@ A bi-encoder maps an entire piece of content (a query, a document page) to a *si
|
||||
|
||||
### Multi-vector (Late-Interaction) Models {#multi-vector-late-interaction-models}
|
||||
|
||||
Multi-vector models, pioneered by ColBERT, take a different approach. Instead of one vector per page, they generate a *set of vectors* for each page—one for every token (or image patch).
|
||||
Multi-vector models, pioneered by ColBERT, take a different approach. Instead of one vector per page, they generate a *set of vectors* for each page: one for every token (or image patch).
|
||||
|
||||
* **Mechanism (MaxSim):** The search process is more sophisticated. For each token in the query, the system finds the most similar token on the page. These maximum similarity scores are then summed up to get the final relevance score. This "late-interaction" preserves fine-grained, token-level details.
|
||||
* **The Models:** We used several vision-language models adapted for this architecture, including `ColPali`, `ColQwen2`, and `ColSmol`. While their underlying transformer backbones differ, they all share the ColBERT philosophy of representing documents as a bag of contextualized token embeddings.
|
||||
|
||||
@@ -4,8 +4,8 @@ sidebarTitle: "NER Search"
|
||||
description: "Extract named entities and embed them to sharpen retrieval."
|
||||
---
|
||||
|
||||
Uses Named Entity Recognition to power vector search. Extracting entities --
|
||||
people, places, organizations -- and embedding them alongside the text improves
|
||||
Uses Named Entity Recognition to power vector search. Extracting entities
|
||||
(people, places, organizations) and embedding them alongside the text improves
|
||||
accuracy when queries are about specific things rather than general topics.
|
||||
|
||||
<Note>
|
||||
|
||||
@@ -6,7 +6,7 @@ description: "Embed video with V-JEPA and search it by similarity."
|
||||
|
||||
Builds a video search application on V-JEPA (Video Joint Embedding Predictive
|
||||
Architecture). Generates vector embeddings for video content and uses
|
||||
[vector search](/search/vector-search) to find visually similar clips -- a
|
||||
[vector search](/search/vector-search) to find visually similar clips: a
|
||||
multimodal case where the searchable column is not text.
|
||||
|
||||
<Note>
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
<parent>
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-parent</artifactId>
|
||||
<version>0.41.0-beta.7</version>
|
||||
<version>0.41.0-beta.9</version>
|
||||
<relativePath>../pom.xml</relativePath>
|
||||
</parent>
|
||||
|
||||
|
||||
+2
-2
@@ -6,7 +6,7 @@
|
||||
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-parent</artifactId>
|
||||
<version>0.41.0-beta.7</version>
|
||||
<version>0.41.0-beta.9</version>
|
||||
<packaging>pom</packaging>
|
||||
<name>${project.artifactId}</name>
|
||||
<description>LanceDB Java SDK Parent POM</description>
|
||||
@@ -28,7 +28,7 @@
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<arrow.version>15.0.0</arrow.version>
|
||||
<lance-core.version>14.0.0-beta.19</lance-core.version>
|
||||
<lance-core.version>14.0.0-beta.21</lance-core.version>
|
||||
<jackson.version>2.18.11</jackson.version>
|
||||
<spotless.skip>false</spotless.skip>
|
||||
<spotless.version>2.30.0</spotless.version>
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
[package]
|
||||
name = "lancedb-nodejs"
|
||||
edition.workspace = true
|
||||
version = "0.41.0-beta.7"
|
||||
version = "0.41.0-beta.9"
|
||||
publish = false
|
||||
license.workspace = true
|
||||
description.workspace = true
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-darwin-arm64",
|
||||
"version": "0.41.0-beta.7",
|
||||
"version": "0.41.0-beta.9",
|
||||
"os": ["darwin"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.darwin-arm64.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-arm64-gnu",
|
||||
"version": "0.41.0-beta.7",
|
||||
"version": "0.41.0-beta.9",
|
||||
"os": ["linux"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.linux-arm64-gnu.node",
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user