rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-12 14:52:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge main into gatekeeper/fix-3350-1

Browse Source 
Preserve fork and namespace pagination regressions, and restore the HashSet import required by remote table pagination.
This commit is contained in:
Gatefixer
2026-10-09 03:19:24 +00:00
parent 132964737a 3348087d09
commit 7e2e791f93
456 changed files with 62298 additions and 12772 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.bumpversion.toml
+1 -1
View File
@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.40.0-beta.12"
current_version = "0.41.0-beta.1"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.
.github/release_notes.json
+5 -1
View File
@@ -6,6 +6,10 @@
"title": "## 🏆 Highlights",
"labels": ["highlight"]
},
{
"title": "## 🚨 Critical Fixes",
"labels": ["critical-fix"]
},
{
"title": "## 🛠 Breaking Changes",
"labels": ["breaking-change"]
@@ -38,4 +42,4 @@
"labels": ["ci"]
}
]
}
}
.github/workflows/license-header-check.yml
+1
View File
@@ -9,6 +9,7 @@ on:
- python/**
- nodejs/**
- java/**
- docs/web-tests/**
- .github/workflows/license-header-check.yml
permissions:
.github/workflows/rust.yml
+4 -35
View File
@@ -285,7 +285,7 @@ jobs:
runs-on: ubuntu-24.04
strategy:
matrix:
msrv: ["1.91.0"] # This should match up with rust-version in Cargo.toml
msrv: ["1.94.0"] # This should match up with rust-version in Cargo.toml
env:
# Need up-to-date compilers for kernels
CC: clang-18
@@ -308,40 +308,9 @@ jobs:
# unreadable outside their own branch anyway, since GitHub scopes
# caches to the creating ref.
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Downgrade dependencies
# These packages have newer requirements for MSRV
run: |
cargo update -p aws-sdk-bedrockruntime --precise 1.77.0
cargo update -p aws-sdk-dynamodb --precise 1.68.0
cargo update -p aws-config --precise 1.6.0
cargo update -p aws-sdk-kms --precise 1.63.0
cargo update -p aws-sdk-s3 --precise 1.79.0
cargo update -p aws-sdk-sso --precise 1.62.0
cargo update -p aws-sdk-ssooidc --precise 1.63.0
cargo update -p aws-sdk-sts --precise 1.63.0
# aws-runtime/sigv4/credential-types/types and the aws-smithy-*
# crates bumped their MSRV to 1.91.1 in late 2026; pin to the last
# 1.91.0-compatible versions. The order matters — each downgrade
# only succeeds once everything that still pins it at a higher
# version has itself been downgraded.
cargo update -p aws-runtime --precise 1.5.12
cargo update -p aws-types --precise 1.3.9
cargo update -p aws-sigv4 --precise 1.3.5
cargo update -p aws-credential-types --precise 1.2.8
# aws-smithy-checksums must stay at or above 0.63.13: OpenDAL's S3
# service needs crc-fast ~1.9, and older releases pin it to ~1.3.
cargo update -p aws-smithy-checksums --precise 0.63.13
cargo update -p aws-smithy-runtime --precise 1.9.3
cargo update -p aws-smithy-http --precise 0.62.6
cargo update -p aws-smithy-eventstream --precise 0.60.14
cargo update -p aws-smithy-http-client --precise 1.1.3
cargo update -p aws-smithy-observability --precise 0.1.4
cargo update -p aws-smithy-query --precise 0.60.8
cargo update -p aws-smithy-runtime-api --precise 1.9.3
cargo update -p aws-smithy-async --precise 1.2.7
cargo update -p aws-smithy-types --precise 1.3.6
cargo update -p aws-smithy-xml --precise 0.60.11
cargo update -p home --precise 0.5.9
- name: Downgrade dependencies
# aws-smithy-types 1.6.1+ requires rustc 1.94.1
run: cargo update -p aws-smithy-types --precise 1.5.0
- name: cargo +${{ matrix.msrv }} check
env:
RUSTUP_TOOLCHAIN: ${{ matrix.msrv }}
.typos.toml
+15 -1
View File
@@ -1,5 +1,15 @@
[default]
extend-ignore-re = ["(?Rm)^.*(#|//)\\s*spellchecker:disable-line$"]
extend-ignore-re = [
"(?Rm)^.*(#|//)\\s*spellchecker:disable-line$",
# Conventional import aliases in the docs examples: `import fiftyone as fo`
# and `import torchvision.io as tio`.
"\\bas (fo|tio)\\b",
"\\btio\\.",
# The substring query in the n-gram full-text search example.
'search\("nce"',
# Garbled OCR output quoted verbatim in the VLM fine-tuning tutorial.
"permission nstallation",
]
[default.extend-words]
# Azure Kubernetes Service, mentioned in rust/lancedb/src/remote/oauth.rs.
@@ -12,6 +22,10 @@ mmaped = "mmaped"
# `WriteableBuffer` is the real name of a type from Python's `_typeshed` stubs,
# used in python/python/lancedb/_blob.py.
Writeable = "Writeable"
# Superlinked's self-hosted inference engine (SIE), its `sie-lancedb` package,
# and the `sie` embedding function it registers, in
# docs/web/integrations/embedding/superlinked.mdx.
SIE = "SIE"
[files]
extend-exclude = [
CONTRIBUTING.md
+44 -1
View File
@@ -71,7 +71,50 @@ are wrappers around the Rust library.
* `nodejs`: Typescript package source code
* `node`: **Deprecated** Typescript package source code
* `java`: Java package source code
* `docs`: Documentation source code
* `docs/src`: SDK reference, built with mkdocs
* `docs/web`: the open-source pages of [docs.lancedb.com](https://docs.lancedb.com)
* `docs/web-tests`: the tests the documentation's code examples are extracted from
## Documentation
The open-source pages of [docs.lancedb.com](https://docs.lancedb.com) live in
`docs/web`, so a change to behaviour and the change to the page describing it
belong in the same pull request.
Preview them locally — this serves `docs/web` on its own, with no other checkout:
```bash
npm i -g mint # https://mintlify.com/docs/installation
cd docs/web && mint dev
```
This repository holds a page at every path the site publishes, so the whole
navigation resolves locally. Enterprise pages exist here as open-source pages --
what the capability is, and what the embedded form does instead. A separate
private repository supplies the fuller version of each, which replaces the page
at the same path when the published site is assembled.
Two things to know before editing:
**Code examples are not written into the pages.** They live in real tests under
`docs/web-tests/{py,ts,rs}`, are extracted into `docs/web/snippets/`, and are
imported by the pages. Edit the test, then regenerate from the repository root:
```bash
uv run docs/web-tests/mdx_snippets_gen.py -s docs/web-tests/py -s docs/web-tests/ts -s docs/web-tests/rs -o docs/web/snippets
```
Review the diff before committing: a few modules currently regenerate with
unrelated changes, so commit only the snippets for the examples you changed.
Run the Python, TypeScript and Rust examples in `docs/web-tests/{py,ts,rs}`
locally before submitting the change. These examples are not yet wired into
pull-request CI; that integration remains follow-up work.
**Every heading carries an explicit `{#anchor}`.** Those anchors are how the
Enterprise pages attach their additions to the right section, so they are
assigned once and never regenerated. Leave an existing anchor alone even when
you reword the heading above it; only new headings need a new one.
## Release process
Cargo.lock
Generated
+63 -63
View File
@@ -1819,7 +1819,7 @@ version = "3.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34"
dependencies = [
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -3087,7 +3087,7 @@ dependencies = [
"libc",
"option-ext",
"redox_users",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -3315,7 +3315,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -3588,8 +3588,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
dependencies = [
"arrow-array",
"rand 0.9.5",
@@ -4880,7 +4880,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
dependencies = [
"hermit-abi",
"libc",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -5143,8 +5143,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
[[package]]
name = "lance"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
dependencies = [
"arc-swap",
"arrow",
@@ -5217,8 +5217,8 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5240,7 +5240,7 @@ dependencies = [
[[package]]
name = "lance-arrow-scalar"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5254,7 +5254,7 @@ dependencies = [
[[package]]
name = "lance-arrow-stats"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5263,8 +5263,8 @@ dependencies = [
[[package]]
name = "lance-bitpacking"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
dependencies = [
"arrayref",
"crunchy",
@@ -5274,8 +5274,8 @@ dependencies = [
[[package]]
name = "lance-core"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5312,8 +5312,8 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
dependencies = [
"arrow",
"arrow-array",
@@ -5344,8 +5344,8 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
dependencies = [
"arrow",
"arrow-array",
@@ -5362,8 +5362,8 @@ dependencies = [
[[package]]
name = "lance-derive"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
dependencies = [
"proc-macro2",
"quote",
@@ -5372,8 +5372,8 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5406,8 +5406,8 @@ dependencies = [
[[package]]
name = "lance-file"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5439,8 +5439,8 @@ dependencies = [
[[package]]
name = "lance-geo"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
dependencies = [
"datafusion",
"geo-traits",
@@ -5454,8 +5454,8 @@ dependencies = [
[[package]]
name = "lance-index"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
dependencies = [
"arc-swap",
"arrow",
@@ -5526,8 +5526,8 @@ dependencies = [
[[package]]
name = "lance-index-core"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5550,8 +5550,8 @@ dependencies = [
[[package]]
name = "lance-io"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
dependencies = [
"arrow",
"arrow-array",
@@ -5593,8 +5593,8 @@ dependencies = [
[[package]]
name = "lance-linalg"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5608,8 +5608,8 @@ dependencies = [
[[package]]
name = "lance-namespace"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
dependencies = [
"arrow",
"async-trait",
@@ -5623,8 +5623,8 @@ dependencies = [
[[package]]
name = "lance-namespace-impls"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
dependencies = [
"arc-swap",
"arrow",
@@ -5678,8 +5678,8 @@ dependencies = [
[[package]]
name = "lance-select"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5693,8 +5693,8 @@ dependencies = [
[[package]]
name = "lance-table"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
dependencies = [
"arrow",
"arrow-array",
@@ -5734,8 +5734,8 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5748,8 +5748,8 @@ dependencies = [
[[package]]
name = "lance-tokenizer"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
dependencies = [
"frostem",
"icu_segmenter",
@@ -5762,7 +5762,7 @@ dependencies = [
[[package]]
name = "lancedb"
version = "0.40.0-beta.12"
version = "0.41.0-beta.1"
dependencies = [
"ahash",
"anyhow",
@@ -5859,7 +5859,7 @@ dependencies = [
[[package]]
name = "lancedb-nodejs"
version = "0.40.0-beta.12"
version = "0.41.0-beta.1"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5884,7 +5884,7 @@ dependencies = [
[[package]]
name = "lancedb-python"
version = "0.40.0-beta.12"
version = "0.41.0-beta.1"
dependencies = [
"arc-swap",
"arrow",
@@ -6637,7 +6637,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -6766,7 +6766,7 @@ version = "5.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
dependencies = [
"base64 0.22.1",
"base64 0.21.7",
"chrono",
"getrandom 0.2.17",
"http 1.5.0",
@@ -8140,8 +8140,8 @@ version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7"
dependencies = [
"heck 0.5.0",
"itertools 0.14.0",
"heck 0.4.1",
"itertools 0.12.1",
"log",
"multimap",
"petgraph",
@@ -8160,7 +8160,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
dependencies = [
"anyhow",
"itertools 0.14.0",
"itertools 0.12.1",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -8422,7 +8422,7 @@ dependencies = [
"once_cell",
"socket2 0.6.3",
"tracing",
"windows-sys 0.60.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -9220,7 +9220,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -9291,7 +9291,7 @@ dependencies = [
"security-framework",
"security-framework-sys",
"webpki-root-certs",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -9838,7 +9838,7 @@ version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451"
dependencies = [
"heck 0.5.0",
"heck 0.4.1",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -9850,7 +9850,7 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40"
dependencies = [
"heck 0.5.0",
"heck 0.4.1",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -10284,7 +10284,7 @@ dependencies = [
"getrandom 0.4.2",
"once_cell",
"rustix",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -11305,7 +11305,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
Cargo.toml
+15 -15
View File
@@ -10,23 +10,23 @@ repository = "https://github.com/lancedb/lancedb"
description = "Serverless, low-latency vector database for AI applications"
keywords = ["lancedb", "lance", "database", "vector", "search"]
categories = ["database-implementations"]
rust-version = "1.91.0"
rust-version = "1.94.0"
[workspace.dependencies]
lance = { "version" = "=13.1.0-beta.1", default-features = false, "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=13.1.0-beta.1", default-features = false, "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=13.1.0-beta.1", default-features = false, "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=14.0.0-beta.7", default-features = false, "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=14.0.0-beta.7", default-features = false, "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=14.0.0-beta.7", default-features = false, "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lancedb = { path = "rust/lancedb", default-features = false }
ahash = "0.8"
# Note that this one does not include pyarrow
README.md
+2 -4
View File
@@ -1,8 +1,6 @@
<a href="https://cloud.lancedb.com" target="_blank">
<img src="https://github.com/user-attachments/assets/92dad0a2-2a37-4ce1-b783-0d1b4f30a00c" alt="LanceDB Cloud Public Beta" width="100%" style="max-width: 100%;">
</a>
<div align="center">
[![Reverie](docs/src/assets/reverie-banner.png)](https://www.reveriesummit.com/?utm_medium=social&utm_source=github&utm_campaign=reverie-2026)
[![LanceDB](docs/src/assets/hero-header.png)](https://lancedb.com)
[![Website](https://img.shields.io/badge/-Website-100000?style=for-the-badge&labelColor=645cfb&color=645cfb)](https://lancedb.com/)
[![Blog](https://img.shields.io/badge/Blog-100000?style=for-the-badge&labelColor=645cfb&color=645cfb)](https://blog.lancedb.com/)
@@ -13,7 +11,7 @@
<img src="docs/src/assets/lancedb.png" alt="LanceDB" width="50%">
# **The Multimodal AI Lakehouse**
# **The Multimodal Lakehouse for AI**
[**How to Install** ](#how-to-install) ✦ [**Detailed Documentation**](https://docs.lancedb.com) ✦ [**Tutorials and Recipes**](https://github.com/lancedb/vectordb-recipes/tree/main) ✦ [**Contributors**](#contributors)
docs/src/assets/hero-header.png
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 1.7 MiB

After

Width:  |  Height:  |  Size: 1.3 MiB

docs/src/assets/reverie-banner.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 722 KiB

docs/src/java/java.md
+1 -1
View File
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
<dependency>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-core</artifactId>
<version>0.40.0-beta.12</version>
<version>0.41.0-beta.1</version>
</dependency>
```
docs/src/js/classes/AutoQuery.md
+2
View File
@@ -417,6 +417,8 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
#### Inherited from
`StandardQueryBase.toArrow`
docs/src/js/classes/Catalog.md
+15 -4
View File
@@ -89,19 +89,30 @@ Drop an empty database. The server rejects nonempty databases.
### listDatabases()
```ts
listDatabases(options): Promise<ListDatabasesResponse>
listDatabases(options): DatabaseNames
```
List one page of databases; pass pageToken from a response for the next page.
Iterate lazily over all database names using `for await...of`.
pageToken resumes from a saved token; omitted starts at the beginning.
pageLimit limits each REST response, not the total; omitted uses the server default.
Request errors are raised during iteration and terminate the iterator.
#### Parameters
* **options** = `{}`
* **options.limit?**: `number`
* **options.pageLimit?**: `number`
* **options.pageToken?**: `string`
#### Returns
`Promise`&lt;[`ListDatabasesResponse`](../interfaces/ListDatabasesResponse.md)&gt;
[`DatabaseNames`](DatabaseNames.md)
#### Example
```ts
for await (const name of catalog.listDatabases({ pageLimit: 20 })) {
console.log(name);
}
```
docs/src/js/classes/Connection.md
+24 -12
View File
@@ -183,10 +183,6 @@ Define a materialized view named `name` over the table `source`.
The view is populated before creation returns. Set `withNoData` to create
only its definition and empty backing table. The view is a normal table:
it can be queried, indexed and searched, and it appears in `tableNames`.
The source table must have stable row ids (create it with
the `newTableEnableStableRowIds` storage option); they keep the view's
provenance valid across source compactions and cannot be enabled after
a table exists.
#### Parameters
@@ -615,30 +611,42 @@ Return true if the connection has not been closed
### listJobs()
```ts
abstract listJobs(): Promise<JobInfo[]>
abstract listJobs(options?): Listing<JobInfo>
```
List server-side jobs across the database's tables.
Iterate lazily over server-side jobs across the database's tables.
Use `for await...of`. Options control page size and the starting token.
See [Listing](Listing.md) for pagination state and error behavior.
#### Parameters
* **options?**: [`ListingOptions`](../interfaces/ListingOptions.md)
#### Returns
`Promise`&lt;[`JobInfo`](../interfaces/JobInfo.md)[]&gt;
[`Listing`](Listing.md)&lt;[`JobInfo`](../interfaces/JobInfo.md)&gt;
***
### listMaterializedViews()
```ts
abstract listMaterializedViews(): Promise<string[]>
abstract listMaterializedViews(options?): Listing<string>
```
The names of the materialized views in this database.
Found by reading every table's schema, so this costs an open per table.
Iterate lazily with `for await...of`. Local listings inspect table schemas
one page at a time. Options control page size and the starting token.
See [Listing](Listing.md) for pagination state and error behavior.
#### Parameters
* **options?**: [`ListingOptions`](../interfaces/ListingOptions.md)
#### Returns
`Promise`&lt;`string`[]&gt;
[`Listing`](Listing.md)&lt;`string`&gt;
***
@@ -739,20 +747,24 @@ A page of table names and an
### listViews()
```ts
abstract listViews(namespacePath?): Promise<string[]>
abstract listViews(namespacePath?, options?): Listing<string>
```
The names of the views in one namespace.
Names only; a definition comes from [describeView](Connection.md#describeview).
Iterate lazily with `for await...of`. See [Listing](Listing.md) for pagination
state and error behavior; options control page size and the starting token.
#### Parameters
* **namespacePath?**: `string`[]
* **options?**: [`ListingOptions`](../interfaces/ListingOptions.md)
#### Returns
`Promise`&lt;`string`[]&gt;
[`Listing`](Listing.md)&lt;`string`&gt;
***
docs/src/js/classes/DatabaseNames.md
+79
View File
@@ -0,0 +1,79 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / DatabaseNames
# Class: DatabaseNames
A lazy async iterator of database names with pagination state.
Returned by [Catalog.listDatabases](Catalog.md#listdatabases).
## Implements
- `AsyncIterableIterator`&lt;`string`&gt;
## Methods
### \[asyncIterator\]()
```ts
asyncIterator: AsyncIterableIterator<string>
```
#### Returns
`AsyncIterableIterator`&lt;`string`&gt;
#### Implementation of
`AsyncIterableIterator.[asyncIterator]`
***
### next()
```ts
next(): Promise<IteratorResult<string, any>>
```
Fetch the next name, requesting another page only when needed.
#### Returns
`Promise`&lt;`IteratorResult`&lt;`string`, `any`&gt;&gt;
#### Implementation of
`AsyncIterableIterator.next`
***
### numPageResults()
```ts
numPageResults(): number
```
Number of names available without another REST request.
#### Returns
`number`
***
### pageToken()
```ts
pageToken(): undefined | string
```
Token for the next REST request. Initially this is the supplied starting token.
Returns undefined after the final page is fetched, even if names remain cached.
Drain the cache before saving a token to avoid skipping names when resuming.
A failed request terminates iteration and retains its token for a new iterator.
#### Returns
`undefined` \| `string`
docs/src/js/classes/Listing.md
+91
View File
@@ -0,0 +1,91 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / Listing
# Class: Listing&lt;T&gt;
A lazy async iterator over database resources with pagination state.
Requests occur during iteration; request errors terminate the iterator.
## Example
```ts
for await (const name of connection.listViews([], { pageLimit: 20 })) {
console.log(name);
}
```
## Type Parameters
• **T**
## Implements
- `AsyncIterableIterator`&lt;`T`&gt;
## Methods
### \[asyncIterator\]()
```ts
asyncIterator: AsyncIterableIterator<T>
```
#### Returns
`AsyncIterableIterator`&lt;`T`&gt;
#### Implementation of
`AsyncIterableIterator.[asyncIterator]`
***
### next()
```ts
next(): Promise<IteratorResult<T, any>>
```
Fetch the next item, requesting another page only when needed.
#### Returns
`Promise`&lt;`IteratorResult`&lt;`T`, `any`&gt;&gt;
#### Implementation of
`AsyncIterableIterator.next`
***
### numPageResults()
```ts
numPageResults(): number
```
Number of cached items available without another request.
#### Returns
`number`
***
### pageToken()
```ts
pageToken(): undefined | string
```
Token for the next request. Initially this is the supplied starting token.
Returns undefined after fetching the final page, even with cached items.
Drain the cache before saving a token to avoid skipping items on resumption.
Failed requests terminate iteration and retain their token for a new iterator.
#### Returns
`undefined` \| `string`
docs/src/js/classes/MaterializedView.md
+3 -10
View File
@@ -65,21 +65,14 @@ refresh(options?): Promise<RefreshMaterializedViewResult>
Recompute the view from its source.
The refresh is incremental when the source's changes can be reconciled
into the view -- rows added, changed or removed since the last one --
and otherwise rebuilds. `full` forces a rebuild; `sourceVersion`
refreshes to that source version instead of the latest.
Concurrent refreshes of one view do not duplicate its rows. Two that
plan the same source rows conflict on commit, and the loser throws
rather than writing them a second time.
Every refresh rebuilds the complete view. For local views,
`sourceVersion` refreshes to that source version instead of the latest;
remote SQL refreshes do not support source-version pinning.
#### Parameters
* **options?**
* **options.full?**: `boolean`
* **options.sourceVersion?**: `number`
#### Returns
docs/src/js/classes/Query.md
+2
View File
@@ -491,6 +491,8 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
#### Inherited from
`StandardQueryBase.toArrow`
docs/src/js/classes/QueryBase.md
+2
View File
@@ -252,6 +252,8 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
***
### withRowId()
docs/src/js/classes/Table.md
+5 -6
View File
@@ -662,10 +662,9 @@ Read the [LsmWriteSpec](../interfaces/LsmWriteSpec.md) currently installed on th
Resolves to `undefined` when the MemWAL LSM write path is not enabled (no
spec has been set, or it was removed with [Table#unsetLsmWriteSpec](Table.md#unsetlsmwritespec)).
The returned spec mirrors what was passed to
[Table#setLsmWriteSpec](Table.md#setlsmwritespec), except that `maintainedIndexes` always
reports the concrete list resolved when the spec was set — `undefined`
never round-trips.
The spec is the one installed, including its maintained-index selection:
an absent `maintainedIndexes` for every index the table has, an empty
array for none.
#### Returns
@@ -1107,8 +1106,8 @@ All variants require the table to have an unenforced primary key
([Table#setUnenforcedPrimaryKey](Table.md#setunenforcedprimarykey)); bucket sharding additionally
requires it to be the single column being bucketed.
Omitting `maintainedIndexes` maintains every index on the table, resolved
here, failing if one cannot be maintained — name them to install anyway.
Omitting `maintainedIndexes` maintains every index the table has,
including ones created later, and skips a kind the MemWAL cannot maintain.
Naming them pins an exact set, and a still-building index is rejected
rather than quietly omitted.
docs/src/js/classes/TakeQuery.md
+2
View File
@@ -267,6 +267,8 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
#### Inherited from
[`QueryBase`](QueryBase.md).[`toArrow`](QueryBase.md#toarrow)
docs/src/js/classes/VectorQuery.md
+4
View File
@@ -447,6 +447,8 @@ nprobes(nprobes): VectorQuery
Set the number of partitions to search (probe)
The number of probes must be greater than 0.
This argument is only used when the vector column has an IVF PQ index.
If there is no index then this value is ignored.
@@ -740,6 +742,8 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
#### Inherited from
`StandardQueryBase.toArrow`
docs/src/js/functions/connectCatalog.md
+3 -1
View File
@@ -28,5 +28,7 @@ headers; opened database connections inherit authentication and client options.
```ts
const catalog = await connectCatalog("https://my-server.example", { apiKey: "secret" });
const db = await catalog.createDatabase("analytics", { existOk: true });
const page = await catalog.listDatabases({ limit: 20 });
for await (const name of catalog.listDatabases({ pageLimit: 20 })) {
console.log(name);
}
```
docs/src/js/globals.md
+6 -1
View File
@@ -27,9 +27,11 @@
- [Branches](classes/Branches.md)
- [Catalog](classes/Catalog.md)
- [Connection](classes/Connection.md)
- [DatabaseNames](classes/DatabaseNames.md)
- [HeaderProvider](classes/HeaderProvider.md)
- [Index](classes/Index.md)
- [Job](classes/Job.md)
- [Listing](classes/Listing.md)
- [MakeArrowTableOptions](classes/MakeArrowTableOptions.md)
- [MatchQuery](classes/MatchQuery.md)
- [MaterializedView](classes/MaterializedView.md)
@@ -108,11 +110,11 @@
- [JobEventsOptions](interfaces/JobEventsOptions.md)
- [JobFailureInfo](interfaces/JobFailureInfo.md)
- [JobInfo](interfaces/JobInfo.md)
- [ListDatabasesResponse](interfaces/ListDatabasesResponse.md)
- [ListNamespacesOptions](interfaces/ListNamespacesOptions.md)
- [ListNamespacesResponse](interfaces/ListNamespacesResponse.md)
- [ListTablesOptions](interfaces/ListTablesOptions.md)
- [ListTablesResponse](interfaces/ListTablesResponse.md)
- [ListingOptions](interfaces/ListingOptions.md)
- [LsmStats](interfaces/LsmStats.md)
- [LsmWriteSpec](interfaces/LsmWriteSpec.md)
- [MaterializedViewDefinition](interfaces/MaterializedViewDefinition.md)
@@ -157,9 +159,12 @@
- [AnalyzePlanDistributedMetrics](type-aliases/AnalyzePlanDistributedMetrics.md)
- [BaseTokenizer](type-aliases/BaseTokenizer.md)
- [BlobData](type-aliases/BlobData.md)
- [BlobInput](type-aliases/BlobInput.md)
- [BlobOptions](type-aliases/BlobOptions.md)
- [BlobRange](type-aliases/BlobRange.md)
- [BlobReadOptions](type-aliases/BlobReadOptions.md)
- [BlobUri](type-aliases/BlobUri.md)
- [Data](type-aliases/Data.md)
- [DataLike](type-aliases/DataLike.md)
- [DataTypeLike](type-aliases/DataTypeLike.md)
docs/src/js/interfaces/FtsOptions.md
+2
View File
@@ -37,6 +37,8 @@ The following tokenizers are available:
"raw" - Raw tokenizer. This tokenizer does not split the text into tokens and indexes the entire text as a single token.
"code" - Code tokenizer. This tokenizer keeps snake_case and camelCase identifiers as single tokens.
"icu" - ICU dictionary-based word segmentation.
"icu/split" - ICU segmentation with simple-style delimiter splitting.
docs/src/js/interfaces/ListDatabasesResponse.md
-23
View File
@@ -1,23 +0,0 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / ListDatabasesResponse
# Interface: ListDatabasesResponse
## Properties
### databases
```ts
databases: string[];
```
***
### pageToken?
```ts
optional pageToken: string;
```
docs/src/js/interfaces/ListTablesOptions.md
+1
View File
@@ -15,6 +15,7 @@ optional limit: number;
```
An upper bound on how many tables to return.
Defaults to 100. Zero returns an empty page without a continuation token.
A page may hold fewer than this and still not be the last one, so keep
going while the response carries a page token rather than while pages are
docs/src/js/interfaces/ListingOptions.md
+29
View File
@@ -0,0 +1,29 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / ListingOptions
# Interface: ListingOptions
Pagination controls for lazy resource listings.
## Properties
### pageLimit?
```ts
optional pageLimit: number;
```
Maximum results per request, not a limit on the whole listing.
***
### pageToken?
```ts
optional pageToken: string;
```
Starting continuation token; omitted starts at the beginning.
docs/src/js/interfaces/LsmWriteSpec.md
+2 -3
View File
@@ -34,9 +34,8 @@ Bucket and identity variants: the sharding column.
optional maintainedIndexes: string[];
```
Indexes the MemWAL keeps up to date. Omit to maintain every supported
index, resolved on install — a snapshot, so indexes created later are not
maintained. Pass `[]` for none.
Indexes the MemWAL keeps up to date. Omit for every index the table has,
including ones created later. Pass `[]` for none.
***
docs/src/js/interfaces/RefreshMaterializedViewResult.md
+1 -1
View File
@@ -14,7 +14,7 @@
mode: string;
```
How the view was brought up to date: "rebuild", "incremental" or "no_op".
How the view was brought up to date: "rebuild".
***
docs/src/js/interfaces/TableNamesOptions.md
+1 -1
View File
@@ -19,7 +19,7 @@ instead.
optional limit: number;
```
An optional limit to the number of results to return.
The maximum number of names to return. Omitted returns all names; zero returns none.
***
docs/src/js/interfaces/TableStatistics.md
+17
View File
@@ -18,6 +18,23 @@ Statistics on table fragments
***
### numDeletedRows?
```ts
optional numDeletedRows: number;
```
The number of rows marked as deleted across all fragments of the table
These rows are not included in `numRows`, but still occupy space on disk
until the table is compacted, so a large value here indicates that the
table should be optimized. Fragments in which every row was deleted are
dropped outright, so their rows are not counted here.
Absent (`undefined`) when the backend does not report deletion counts.
***
### numIndices
```ts
docs/src/js/type-aliases/BaseTokenizer.md
+1
View File
@@ -12,6 +12,7 @@ type BaseTokenizer:
| "whitespace"
| "raw"
| "ngram"
| "code"
| "icu"
| "icu/split"
| `jieba/${string}`
docs/src/js/type-aliases/BlobData.md
+16
View File
@@ -0,0 +1,16 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobData
# Type Alias: BlobData
```ts
type BlobData: Buffer | Uint8Array | ArrayBuffer | Blob;
```
Bytes accepted for a blob value. `ArrayBuffer` is wrapped without copying.
`Blob` and `File` are read with `arrayBuffer()`, which only the async write
paths ([Connection.createTable](../classes/Connection.md#createtable), [Table.add](../classes/Table.md#add),
[Table.mergeInsert](../classes/Table.md#mergeinsert)) can do.
docs/src/js/type-aliases/BlobInput.md
+36
View File
@@ -0,0 +1,36 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobInput
# Type Alias: BlobInput
```ts
type BlobInput:
| BlobData
| BlobUri
| object
| object
| null
| undefined;
```
A value accepted for a `lance.blob.v2` column (see [blob](../functions/blob.md)).
Either inline bytes, a URI pointing at external bytes, or a struct that sets
exactly one of `data` or `uri`. `null` and `undefined` write a null blob.
## Example
```ts
import { pathToFileURL } from "node:url";
import type { BlobInput } from "@lancedb/lancedb";
const rows: { id: bigint; image: BlobInput }[] = [
{ id: 1n, image: await (await fetch(url)).arrayBuffer() },
{ id: 2n, image: pathToFileURL("/data/cat.png") },
{ id: 3n, image: { data: new Blob(["hello"]) } },
];
await table.add(rows);
```
docs/src/js/type-aliases/BlobUri.md
+13
View File
@@ -0,0 +1,13 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobUri
# Type Alias: BlobUri
```ts
type BlobUri: string | URL;
```
A URI accepted for a blob value. A `URL` is stored as its `href`.
docs/src/python/python.md
+25 -28
View File
@@ -26,6 +26,14 @@ is also an [asynchronous API client](#connections-asynchronous).
::: lancedb.db.DBConnection
Resource listings (`list_secrets`, `list_views`, `list_jobs`, `list_functions`, and
`list_materialized_views`) return lazy iterators. Use `list(db.list_views())` to
collect synchronous results, or `[name async for name in db.list_views()]` with
an asynchronous connection. `page_limit` controls each request, and `page_token`
resumes a listing. Drain the iterator's cached results before saving its token.
::: lancedb.listing.Listing
::: lancedb.Session
## Catalogs (Synchronous)
@@ -37,7 +45,8 @@ are ordinary connections. Dropping a database requires it to be empty.
::: lancedb.catalog.Catalog
::: lancedb.catalog.ListDatabasesResponse
::: lancedb.catalog.DatabaseNames
## Remote SQL
@@ -77,6 +86,15 @@ print(query.describe().status)
for batch in query.reader():
print(batch.num_rows)
# Values bind to $1 / $name placeholders and travel as Arrow, so a float32
# stays a float32 and a vector stays a compact fixed-size list:
import numpy as np
reader = db.execute_query(
"SELECT id FROM docs ORDER BY distance(vector, $vector) LIMIT $k",
parameters={"vector": np.random.rand(768).astype(np.float32), "k": 10},
)
# The async connection exposes the same lifecycle without blocking:
# async_db = await lancedb.connect_async(
# "db://analytics",
@@ -192,39 +210,14 @@ listing a storage directory.
::: lancedb.sql.QueryDescription
::: lancedb.sql.QueryParameters
## Materialized Views (Synchronous)
::: lancedb.materialized_view.MaterializedView
::: lancedb.materialized_view.MaterializedViewDefinition
::: lancedb.vector_dedup
::: lancedb.materialized_view.VectorDedupSource
For an existing indexed source, declare and materialize a dedup result without
writing SQL. Creation captures the source version once and waits for the complete
result; remote connections use the service's MV jobs.
```python
from lancedb import vector_dedup
view = db.create_materialized_view(
"images_clean",
vector_dedup("images", column="phash", distance_threshold=4),
)
cleaned = view.table
```
To submit without waiting, call `db.create_materialized_view_async(...)` with the
same source expression and wait on the returned job. To declare only, pass
`with_no_data=True`, then use `view.refresh_async().wait()` when ready. Pass
`dataset_version=...` to `vector_dedup` for an explicit snapshot. The default
policy retains direct representatives (A-B-C without A-C keeps A and C), preserves
the source, and materializes its original columns. Only within-segment/partition
pairs are covered, using the index's distance representation.
## Views
::: lancedb.view.ViewDescription
@@ -440,6 +433,8 @@ latency on your workload before opting in.
::: lancedb.catalog.AsyncCatalog
::: lancedb.catalog.AsyncDatabaseNames
Connections represent a connection to a LanceDb database and
can be used to create, list, or open tables.
@@ -447,6 +442,8 @@ can be used to create, list, or open tables.
::: lancedb.db.AsyncConnection
::: lancedb.listing.AsyncListing
## Namespaces (Asynchronous)
::: lancedb.connect_namespace_async
docs/web-tests/camelot.json
+66
View File
@@ -0,0 +1,66 @@
[
{
"id": 1,
"name": "King Arthur",
"role": "King of Camelot",
"description": "The legendary ruler of Camelot, wielder of Excalibur, and leader of the Knights of the Round Table.",
"vector": [0.72, -0.28, 0.60, 0.86],
"stats": { "strength": 2, "courage": 5, "magic": 1, "wisdom": 4 }
},
{
"id": 2,
"name": "Merlin",
"role": "Wizard and Advisor",
"description": "A powerful wizard and prophet who mentors Arthur and shapes the destiny of Camelot through magic and foresight.",
"vector": [0.05, 0.88, 0.62, 0.85],
"stats": { "strength": 2, "courage": 4, "magic": 5, "wisdom": 5 }
},
{
"id": 3,
"name": "Queen Guinevere",
"role": "Queen of Camelot",
"description": "Arthur's queen, admired for her grace and diplomacy, whose romances and loyalties influence Camelot's fate.",
"vector": [0.22, -0.22, 0.42, 0.82],
"stats": { "strength": 1, "courage": 3, "magic": 1, "wisdom": 4 }
},
{
"id": 4,
"name": "Sir Lancelot",
"role": "Knight of the Round Table",
"description": "Arthur's most skilled knight, famed for unmatched combat prowess and his tragic love for Queen Guinevere.",
"vector": [0.86, -0.35, 0.38, 0.55],
"stats": { "strength": 5, "courage": 5, "magic": 1, "wisdom": 3 }
},
{
"id": 5,
"name": "Sir Gawain",
"role": "Knight of the Round Table",
"description": "A noble and honorable knight known for his courtesy and his encounter with the Green Knight.",
"vector": [0.82, -0.32, 0.52, 0.60],
"stats": { "strength": 4, "courage": 5, "magic": 1, "wisdom": 4 }
},
{
"id": 6,
"name": "Sir Galahad",
"role": "Knight of the Round Table",
"description": "The purest and most virtuous knight, chosen to achieve the Holy Grail due to his unwavering spiritual purity.",
"vector": [0.80, -0.20, 0.70, 0.78],
"stats": { "strength": 4, "courage": 5, "magic": 2, "wisdom": 5 }
},
{
"id": 7,
"name": "Sir Percival",
"role": "Knight of the Round Table",
"description": "A loyal and innocent knight whose bravery and sincerity make him one of the key seekers of the Holy Grail.",
"vector": [0.78, -0.36, 0.48, 0.52],
"stats": { "strength": 4, "courage": 4, "magic": 1, "wisdom": 3 }
},
{
"id": 8,
"name": "Mordred",
"role": "Traitor Knight",
"description": "Arthur's treacherous son or nephew who ultimately rebels against him, leading to Camelot's downfall.",
"vector": [0.68, -0.30, -0.65, 0.20],
"stats": { "strength": 4, "courage": 2, "magic": 1, "wisdom": 2 }
}
]
docs/web-tests/mdx_snippets_gen.py
+362
View File
@@ -0,0 +1,362 @@
"""
Generate MDX-ready snippet modules that export raw string constants.
The generator groups snippets by the originating test file name rather than by
language. For a source file like `tests/py/test_basic_usage.py`, the snippets are
written under `docs/snippets/basic.mdx`. Each exported constant is prefixed with
a language identifier (Py|Ts|Rs) followed by the snippet name converted to
TitleCase so that docs authors can selectively assemble `<CodeGroup>` blocks.
"""
from __future__ import annotations
import argparse
import json
import os
import re
from dataclasses import dataclass
from pathlib import Path
from typing import Dict, Iterable, Iterator, List, Mapping, MutableMapping, Tuple
# Supported languages and relevant metadata.
LANG_ORDER = ("py", "ts", "rs")
LANG_PREFIX = {"py": "Py", "ts": "Ts", "rs": "Rs"}
LANG_EXTENSIONS = {
".py": "py",
".ts": "ts",
".tsx": "ts",
".rs": "rs",
}
LANG_MARKERS = {"py": ["#"], "ts": ["//"], "rs": ["//"]}
# Match each export block; we rely on the blank line we emit after each export to
# avoid accidentally truncating on semicolons inside the JSON string literal.
EXPORT_RE = re.compile(r"export const (\w+)\s*=\s*(.+?);\s*\n(?:\s*\n|$)", re.DOTALL)
DEFAULT_SOURCE_DIRS = (
Path("tests") / "py",
Path("tests") / "ts",
Path("tests") / "rs",
)
@dataclass(frozen=True)
class SnippetRecord:
lang: str
snippet_name: str
export_name: str
text: str
source_rel: str
def build_markers(
comment_markers: Iterable[str],
) -> Tuple[re.Pattern[str], re.Pattern[str]]:
escaped = "|".join(re.escape(c) for c in comment_markers)
start_re = re.compile(rf"^\s*(?:{escaped})\s*--8<--\s*\[start:([^\]]+)\]")
end_re = re.compile(rf"^\s*(?:{escaped})\s*--8<--\s*\[end:([^\]]+)\]")
return start_re, end_re
def parse_snippets(
lines: List[str], start_re: re.Pattern[str], end_re: re.Pattern[str]
) -> Dict[str, List[str]]:
snippets: Dict[str, List[str]] = {}
stack: List[Tuple[str, int]] = []
for idx, line in enumerate(lines):
m_start = start_re.match(line)
if m_start:
name = m_start.group(1).strip()
stack.append((name, idx))
continue
m_end = end_re.match(line)
if m_end:
name = m_end.group(1).strip()
if not stack:
raise ValueError(
f"End marker for '{name}' found at line {idx + 1} without matching start"
)
start_name, start_idx = stack.pop()
if start_name != name:
raise ValueError(
f"Mismatched markers: start '{start_name}' at line {start_idx + 1}, "
f"end '{name}' at line {idx + 1}"
)
snippets[name] = lines[start_idx + 1 : idx]
continue
if stack:
open_names = ", ".join(f"{n}@{i + 1}" for n, i in stack)
raise ValueError(f"Unclosed snippet markers for: {open_names}")
return snippets
def dedent(lines: List[str]) -> List[str]:
min_indent = None
for ln in lines:
if ln.strip() == "":
continue
expanded = ln.expandtabs(4)
indent = len(expanded) - len(expanded.lstrip(" "))
if min_indent is None or indent < min_indent:
min_indent = indent
if not lines:
return []
if min_indent in (None, 0):
return [ln.rstrip("\n") for ln in lines]
dedented: List[str] = []
for ln in lines:
expanded = ln.expandtabs(4)
if expanded.strip() == "":
dedented.append(expanded.rstrip("\n"))
else:
dedented.append(expanded[min_indent:].rstrip("\n"))
return dedented
def ensure_dir(path: Path) -> None:
path.mkdir(parents=True, exist_ok=True)
def detect_language(file_path: Path) -> str:
lang = LANG_EXTENSIONS.get(file_path.suffix.lower())
if not lang:
raise ValueError(f"Unsupported file extension for {file_path}")
return lang
def markers_for_lang(lang: str) -> Tuple[re.Pattern[str], re.Pattern[str]]:
markers = LANG_MARKERS.get(lang)
if not markers:
raise ValueError(f"Unsupported language for markers: {lang}")
return build_markers(markers)
def normalize_target_name(file_path: Path) -> str:
stem = file_path.stem
if stem.startswith("test_"):
stem = stem[len("test_") :]
if stem.endswith("_test"):
stem = stem[: -len("_test")]
if stem.endswith(".test"):
stem = stem[: -len(".test")]
if stem.endswith(".spec"):
stem = stem[: -len(".spec")]
if stem == "":
stem = file_path.stem
return stem.replace(" ", "_")
def to_title_case(name: str) -> str:
parts = re.split(r"[^0-9a-zA-Z]+", name)
filtered = [p for p in parts if p]
return "".join(p[:1].upper() + p[1:].lower() for p in filtered) or "Snippet"
def format_export_name(lang: str, snippet_name: str) -> str:
prefix = LANG_PREFIX[lang]
return f"{prefix}{to_title_case(snippet_name)}"
def snippet_text(lines: List[str]) -> str:
if not lines:
return ""
joined = "\n".join(lines).rstrip("\n")
return (joined + "\n") if joined else ""
def to_js_literal(text: str) -> str:
return json.dumps(text, ensure_ascii=False)
def split_export_prefix(export_name: str) -> Tuple[str | None, str]:
for lang, prefix in LANG_PREFIX.items():
if export_name.startswith(prefix):
suffix = export_name[len(prefix) :]
return lang, suffix or export_name
return None, export_name
def parse_existing_module(path: Path) -> Mapping[str, Mapping[str, SnippetRecord]]:
try:
text = path.read_text(encoding="utf-8")
except FileNotFoundError:
return {}
existing: Dict[str, Dict[str, SnippetRecord]] = {}
for match in EXPORT_RE.finditer(text):
export_name, literal = match.group(1), match.group(2).strip()
lang, snippet_suffix = split_export_prefix(export_name)
if not lang:
continue
try:
decoded = json.loads(literal)
except Exception:
decoded = literal.strip().strip(";")
record = SnippetRecord(
lang=lang,
snippet_name=snippet_suffix or export_name,
export_name=export_name,
text=decoded,
source_rel="(existing module)",
)
existing.setdefault(lang, {})[record.snippet_name] = record
return existing
def iter_source_files(source_dirs: Iterable[Path]) -> Iterator[Path]:
for root in source_dirs:
for path in root.rglob("*"):
if path.is_file() and path.suffix.lower() in LANG_EXTENSIONS:
yield path
def write_if_changed(path: Path, content: str) -> bool:
if path.exists():
try:
current = path.read_text(encoding="utf-8")
if current == content:
return False
except Exception:
pass
ensure_dir(path.parent)
path.write_text(content, encoding="utf-8")
return True
def collect_snippets(
source_dirs: Iterable[Path],
) -> Mapping[str, Mapping[str, Mapping[str, SnippetRecord]]]:
result: Dict[str, Dict[str, Dict[str, SnippetRecord]]] = {}
for file_path in iter_source_files(source_dirs):
lang = detect_language(file_path)
start_re, end_re = markers_for_lang(lang)
try:
text = file_path.read_text(encoding="utf-8")
except Exception as exc: # pragma: no cover
raise RuntimeError(f"Failed to read {file_path}: {exc}") from exc
lines = text.splitlines(keepends=True)
snippets = parse_snippets(lines, start_re, end_re)
if not snippets:
continue
target = normalize_target_name(file_path)
lang_map = result.setdefault(target, {}).setdefault(lang, {})
for snippet_name, content_lines in snippets.items():
if snippet_name in lang_map:
raise ValueError(
f"Duplicate snippet '{snippet_name}' in {file_path} and {lang_map[snippet_name].source_rel}"
)
dedented = dedent(content_lines)
export_name = format_export_name(lang, snippet_name)
source_rel = os.path.relpath(file_path, Path.cwd())
lang_map[snippet_name] = SnippetRecord(
lang=lang,
snippet_name=snippet_name,
export_name=export_name,
text=snippet_text(dedented),
source_rel=source_rel,
)
return result
def render_module(
target: str, lang_map: Mapping[str, Mapping[str, SnippetRecord]]
) -> str:
parts: List[str] = []
parts.append(
"{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */}\n\n"
)
for lang in LANG_ORDER:
snippets = lang_map.get(lang)
if not snippets:
continue
for snippet_name in sorted(snippets.keys()):
record = snippets[snippet_name]
literal = to_js_literal(record.text)
parts.append(f"export const {record.export_name} = {literal};\n\n")
return "".join(parts)
def generate_modules(
snippets_by_target: Mapping[str, Mapping[str, Mapping[str, SnippetRecord]]],
output_root: Path,
) -> None:
modules_written = 0
total_exports = 0
for target in sorted(snippets_by_target.keys()):
module_path = output_root / f"{target}.mdx"
lang_map = snippets_by_target[target]
existing_langs = parse_existing_module(module_path)
for lang, snippets in existing_langs.items():
if lang in lang_map:
continue
print(module_path, " -- ", lang)
merged_lang = lang_map.setdefault(lang, {})
existing_export_names = {rec.export_name for rec in merged_lang.values()}
for snippet_name, record in snippets.items():
# If an export with the same name already exists (e.g., regenerated),
# prefer the newly generated snippet and skip the old one to avoid duplicates.
if record.export_name in existing_export_names:
continue
merged_lang.setdefault(snippet_name, record)
total_exports += sum(len(snippets) for snippets in lang_map.values())
module_content = render_module(target, lang_map)
if write_if_changed(module_path, module_content):
modules_written += 1
print(
f"\nGenerated {total_exports} snippets\n---"
)
def resolve_source_dirs(args_dirs: List[str] | None) -> List[Path]:
if args_dirs:
dirs = [Path(p) for p in args_dirs]
else:
dirs = list(DEFAULT_SOURCE_DIRS)
missing = [str(p) for p in dirs if not p.exists()]
if missing:
raise FileNotFoundError(f"Source directories not found: {', '.join(missing)}")
return dirs
def main() -> int:
parser = argparse.ArgumentParser(
description="Generate MDX snippet modules grouped by test filename."
)
parser.add_argument(
"-s",
"--source-dir",
action="append",
help="Path(s) to language test directories (default: tests/py, ts, rs)",
)
parser.add_argument(
"-o",
"--output-dir",
default=str(Path("docs") / "snippets"),
help="Directory where snippet modules are written (default: docs/snippets)",
)
args = parser.parse_args()
source_dirs = resolve_source_dirs(args.source_dir)
snippets = collect_snippets(source_dirs)
if not snippets:
print("No snippets found.")
return 0
output_root = Path(args.output_dir)
generate_modules(snippets, output_root)
return 0
if __name__ == "__main__":
raise SystemExit(main())
docs/web-tests/py/conftest.py
+56
View File
@@ -0,0 +1,56 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import shutil
from pathlib import Path
import lancedb
import pytest
import pytest_asyncio
from lancedb.db import AsyncConnection, DBConnection
class DatabasePathFactory:
"""Create per-test database directories and ensure they get removed."""
def __init__(self, base_dir: Path) -> None:
self._base_dir = base_dir
self._created: list[Path] = []
def __call__(self, name: str) -> Path:
safe_name = name.replace("/", "_")
path = self._base_dir / safe_name
if path.exists():
shutil.rmtree(path, ignore_errors=True)
path.mkdir(parents=True, exist_ok=True)
self._created.append(path)
return path
def cleanup(self) -> None:
for path in reversed(self._created):
shutil.rmtree(path, ignore_errors=True)
self._created.clear()
@pytest.fixture
def db_path_factory(tmp_path: Path):
factory = DatabasePathFactory(tmp_path)
yield factory
factory.cleanup()
@pytest.fixture
def mem_db() -> DBConnection:
return lancedb.connect("memory://")
@pytest.fixture
def tmp_db(db_path_factory) -> DBConnection:
"""Create a temporary database connection for testing."""
db_path = db_path_factory("tmp_db")
return lancedb.connect(str(db_path))
@pytest_asyncio.fixture
async def mem_db_async() -> AsyncConnection:
return await lancedb.connect_async("memory://")
docs/web-tests/py/test_basic_usage.py
+205
View File
@@ -0,0 +1,205 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
# --8<-- [start:basic_imports]
import json
import lancedb
import pandas as pd
import polars as pl
import pyarrow as pa
# --8<-- [end:basic_imports]
import pytest
data_path = "tests/camelot.json"
def test_basic_usage(db_path_factory):
uri = "./basic_usage_db"
uri = db_path_factory("basic_usage_db")
db = lancedb.connect(uri)
# --8<-- [start:data_load]
with open(data_path, "r") as f:
data = json.load(f)
# --8<-- [end:data_load]
# --8<-- [start:basic_create_table]
table = db.create_table("camelot", data=data, mode="overwrite")
# --8<-- [end:basic_create_table]
assert len(table) == 8
# --8<-- [start:basic_open_table]
table = db.open_table("camelot")
# --8<-- [end:basic_open_table]
# --8<-- [start:basic_create_table_pandas]
pandas_df = pd.DataFrame(data)
table_pd = db.create_table("camelot_pd", data=pandas_df, mode="overwrite")
# --8<-- [end:basic_create_table_pandas]
assert len(table_pd) == 8
db.drop_table("camelot_pd")
# --8<-- [start:basic_create_table_polars]
polars_df = pl.DataFrame(data)
table_pl = db.create_table("camelot_pl", data=polars_df, mode="overwrite")
# --8<-- [end:basic_create_table_polars]
assert len(table_pl) == 8
db.drop_table("camelot_pl")
# --8<-- [start:basic_create_empty_table]
schema = pa.schema(
[
pa.field("id", pa.uint16()),
pa.field("name", pa.string()),
pa.field("role", pa.string()),
pa.field("description", pa.string()),
pa.field("vector", pa.list_(pa.float32(), 4)),
pa.field(
"stats",
pa.struct(
[
pa.field("strength", pa.int8()),
pa.field("courage", pa.int8()),
pa.field("magic", pa.int8()),
pa.field("wisdom", pa.int8()),
]
),
),
]
)
db.create_table("camelot_pa", schema=schema, mode="overwrite")
# --8<-- [end:basic_create_empty_table]
assert "camelot_pa" in db.list_tables().tables
db.drop_table("camelot_pa")
# --8<-- [start:basic_add_data]
magical_characters = [
{
"id": 9,
"name": "Morgan le Fay",
"role": "Sorceress",
"description": "A powerful enchantress, Arthur's half-sister, and a complex figure who oscillates between aiding and opposing Camelot.",
"vector": [0.10, 0.84, 0.25, 0.70],
"stats": { "strength": 2, "courage": 3, "magic": 5, "wisdom": 4 }
},
{
"id": 10,
"name": "The Lady of the Lake",
"role": "Mystical Guardian",
"description": "A mysterious supernatural figure associated with Avalon, known for giving Arthur the sword Excalibur.",
"vector": [0.00, 0.90, 0.58, 0.88],
"stats": { "strength": 2, "courage": 3, "magic": 5, "wisdom": 5 }
}
]
table.add(magical_characters)
# --8<-- [end:basic_add_data]
assert len(table) == 10
# --8<-- [start:basic_vector_search]
query_vector = [0.03, 0.85, 0.61, 0.90]
result = table.search(query_vector).limit(5).to_polars()
print(result)
# --8<-- [end:basic_vector_search]
# --8<-- [start:basic_add_columns]
table.add_columns(
{
"power": "cast(((stats.strength + stats.courage + stats.magic + stats.wisdom) / 4.0) as float)"
}
)
# --8<-- [end:basic_add_columns]
assert "power" in table.schema.names
# Run examples to illustrate search
# --8<-- [start:basic_vector_search_q1]
# Who are the characters similar to "wizard"?
query_vector_1 = [0.03, 0.85, 0.61, 0.90]
r1 = (
table.search(query_vector_1)
.limit(5)
.select(["name", "role", "description"])
.to_polars()
)
print(r1)
# --8<-- [end:basic_vector_search_q1]
# --8<-- [start:basic_vector_search_q2]
# Who are the characters with high magic stats?
query_vector_2 = [0.03, 0.85, 0.61, 0.90]
r2 = (
table.search(query_vector_2)
.where("stats.magic > 3")
.select(["name", "role", "description"])
.limit(5)
.to_polars()
)
print(r2)
# --8<-- [end:basic_vector_search_q2]
# --8<-- [start:basic_vector_search_q3]
# Who are the strongest characters?
r3 = (
table.search()
.where("stats.strength > 3")
.select(["name", "role", "description"])
.limit(5)
.to_polars()
)
print(r3)
# --8<-- [end:basic_vector_search_q3]
# --8<-- [start:basic_vector_search_q4]
# Who are the strongest characters?
r4 = (
table.search()
.select(["name", "role", "description", "power"])
.to_polars()
)
print(r4)
# --8<-- [end:basic_vector_search_q4]
# --8<-- [start:basic_sort_polars]
# Sort Polars DataFrame by power in descending order
print(r4.sort("power", descending=True).limit(5))
# --8<-- [end:basic_sort_polars]
sorted_power = r4.sort("power", descending=True)["power"].to_list()
assert sorted_power == sorted(sorted_power, reverse=True)
assert sorted_power[0] == 4.0
# --8<-- [start:basic_drop_columns]
table.drop_columns(["power"])
# --8<-- [end:basic_drop_columns]
# --8<-- [start:basic_delete_rows]
table.delete('role = "Traitor Knight"')
# --8<-- [end:basic_delete_rows]
assert len(table) == 9
# --8<-- [start:basic_drop_table]
db.drop_table("camelot")
# --8<-- [end:basic_drop_table]
assert "camelot" not in db.list_tables().tables
@pytest.mark.asyncio
async def test_basic_usage_async_api(db_path_factory):
uri = db_path_factory("basic_usage_async_db")
with open(data_path, "r") as f:
data = json.load(f)
# --8<-- [start:basic_async_api]
import lancedb
async_db = await lancedb.connect_async(uri)
async_table = await async_db.create_table(
"camelot_async",
data=data,
mode="overwrite",
)
query_vector = [0.03, 0.85, 0.61, 0.90]
async_results = await (
await async_table.search(query_vector)
).limit(5).select(["name", "role", "description"]).to_polars()
print(async_results)
# --8<-- [end:basic_async_api]
assert async_results.height >= 1
docs/web-tests/py/test_build_with_ai_agents.py
+109
View File
@@ -0,0 +1,109 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
# --8<-- [start:camelot_schema]
from lancedb.pydantic import LanceModel
from pydantic import ConfigDict
class Stats(LanceModel):
model_config = ConfigDict(strict=True, extra="forbid")
strength: int
courage: int
magic: int
wisdom: int
class Character(LanceModel):
model_config = ConfigDict(strict=True, extra="forbid")
id: int
name: str
role: str
description: str
stats: Stats
image_filename: str
image: bytes
# --8<-- [end:camelot_schema]
# --8<-- [start:camelot_batches]
import json
from collections.abc import Iterator
from pathlib import Path
def validated_batches(
input_path: Path, batch_size: int
) -> Iterator[list[dict]]:
raw_records = json.loads(input_path.read_text())
asset_root = input_path.parent.parent
batch: list[dict] = []
for raw in raw_records:
payload = dict(raw)
image_path = asset_root / payload.pop("img")
payload["image_filename"] = image_path.name
payload["image"] = image_path.read_bytes()
character = Character.model_validate(payload)
batch.append(character.model_dump(mode="python"))
if len(batch) == batch_size:
yield batch
batch = []
if batch:
yield batch
# --8<-- [end:camelot_batches]
# --8<-- [start:camelot_oss_ingestion]
import lancedb
def ingest_oss(
input_path: Path,
uri: str = "data/camelot.lancedb",
table_name: str = "camelot_multimodal",
batch_size: int = 64,
):
db = lancedb.connect(uri)
if table_name in db.list_tables():
raise ValueError(
f"Table {table_name!r} already exists. Choose a fresh table name."
)
table = db.create_table(table_name, schema=Character)
for batch in validated_batches(input_path, batch_size):
table.add(batch)
table.optimize()
return table
if __name__ == "__main__":
ingest_oss(Path("data/camelot.json"))
# --8<-- [end:camelot_oss_ingestion]
def test_camelot_oss_ingestion(tmp_path):
source = Path(
"docs/web/static/assets/tutorials/build-with-ai-agents/camelot/data/camelot.json"
)
table = ingest_oss(
source,
uri=str(tmp_path / "camelot.lancedb"),
table_name="camelot",
)
rows = (
table.search()
.select(["id", "name", "image", "image_filename"])
.limit(8)
.to_list()
)
assert len(rows) == 8
assert all(row["image"] for row in rows)
assert all(row["image_filename"].endswith(".jpg") for row in rows)
docs/web-tests/py/test_connection.py
+148
View File
@@ -0,0 +1,148 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
from pathlib import Path
import shutil
def test_connection():
# --8<-- [start:connect]
import lancedb
uri = "ex_lancedb"
db = lancedb.connect(uri)
# --8<-- [end:connect]
assert db is not None
shutil.rmtree(uri, ignore_errors=True)
assert not Path(uri).exists()
async def connect_async_example():
# --8<-- [start:connect_async]
import lancedb
uri = "ex_lancedb"
async_db = await lancedb.connect_async(uri)
# --8<-- [end:connect_async]
return async_db
def connect_enterprise_quickstart_config():
import lancedb
# --8<-- [start:connect_enterprise_quickstart]
uri = "db://your-database-uri"
api_key = "your-api-key"
region = "us-east-1"
host_override = "https://your-enterprise-endpoint.com"
db = lancedb.connect(
uri=uri,
api_key=api_key,
region=region,
host_override=host_override,
)
# --8<-- [end:connect_enterprise_quickstart]
return db
def test_connect_enterprise_quickstart(monkeypatch):
import lancedb
captured = {}
def fake_connect(**kwargs):
captured.update(kwargs)
return object()
monkeypatch.setattr(lancedb, "connect", fake_connect)
db = connect_enterprise_quickstart_config()
assert db is not None
assert captured == {
"uri": "db://your-database-uri",
"api_key": "your-api-key",
"region": "us-east-1",
"host_override": "https://your-enterprise-endpoint.com",
}
def connect_object_storage_config():
# --8<-- [start:connect_object_storage]
import lancedb
uri = "s3://your-bucket/path"
# You can also use "gs://your-bucket/path" or "az://your-container/path".
db = lancedb.connect(uri)
# --8<-- [end:connect_object_storage]
return db
def namespace_table_ops_example():
# --8<-- [start:namespace_table_ops]
import lancedb
db = lancedb.connect_namespace("dir", {"root": "./local_lancedb"})
# Create namespace tree: prod/search
db.create_namespace(["prod"], mode="exist_ok")
db.create_namespace(["prod", "search"], mode="exist_ok")
db.create_namespace(["prod", "recommendations"], mode="exist_ok")
db.create_table(
"user",
data=[{"id": 1, "vector": [0.1, 0.2], "name": "alice"}],
namespace_path=["prod", "search"],
mode="create", # use "overwrite" only if you want to replace existing table
)
db.create_table(
"user",
data=[{"id": 2, "vector": [0.3, 0.4], "name": "bob"}],
namespace_path=["prod", "recommendations"],
mode="create", # use "overwrite" only if you want to replace existing table
)
# Verify
print(db.list_namespaces()) # ['prod']
print(db.list_namespaces(namespace_path=["prod"])) # ['recommendations', 'search']
print(db.list_tables(namespace_path=["prod", "search"])) # ['user']
print(db.list_tables(namespace_path=["prod", "recommendations"])) # ['user']
# --8<-- [end:namespace_table_ops]
def namespace_admin_ops_example():
# --8<-- [start:namespace_admin_ops]
import lancedb
db = lancedb.connect_namespace("dir", {"root": "./local_lancedb"})
namespace = ["prod", "search"]
db.create_namespace(["prod"])
db.create_namespace(["prod", "search"])
child_namespaces = db.list_namespaces(namespace_path=["prod"]).namespaces
print(f"Child namespaces under {namespace}: {child_namespaces}")
# Child namespaces under ['prod', 'search']: ['search']
metadata = db.describe_namespace(["prod", "search"])
print(f"Metadata for namespace {namespace}: {metadata}")
# Metadata for namespace ['prod', 'search']: properties=None
db.drop_namespace(["prod", "search"], mode="skip")
db.drop_namespace(["prod"], mode="skip")
# --8<-- [end:namespace_admin_ops]
return child_namespaces, metadata
async def connect_object_storage_config_async():
# --8<-- [start:connect_object_storage_async]
import lancedb
uri = "s3://your-bucket/path"
# You can also use "gs://your-bucket/path" or "az://your-container/path".
async_db = await lancedb.connect_async(uri)
# --8<-- [end:connect_object_storage_async]
return async_db
docs/web-tests/py/test_doc_function_examples.py
+34
View File
@@ -0,0 +1,34 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
from lancedb.index import FTS
def test_python_boolean_query_operators(tmp_db):
table = tmp_db.create_table(
"boolean_docs",
[
{"id": 1, "text": "puppy"},
{"id": 2, "text": "merrily"},
{"id": 3, "text": "puppy merrily"},
{"id": 4, "text": "other"},
],
)
table.create_index("text", config=FTS())
# --8<-- [start:fts_boolean_queries]
from lancedb.query import MatchQuery
intersection = MatchQuery("puppy", "text") & MatchQuery("merrily", "text")
union = MatchQuery("puppy", "text") | MatchQuery("merrily", "text")
# --8<-- [end:fts_boolean_queries]
def ids(query):
return set(table.search(query, query_type="fts").to_arrow()["id"].to_pylist())
assert ids(intersection) == {3}
assert ids(union) == {1, 2, 3}
a = MatchQuery("puppy", "text")
b = MatchQuery("merrily", "text")
assert ids(a and b) == {2, 3}
assert ids(a or b) == {1, 3}
docs/web-tests/py/test_doc_storage_examples.py
+84
View File
@@ -0,0 +1,84 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import lancedb
import numpy as np
import pyarrow as pa
import pytest
def test_rebuild_vector_index(tmp_db):
rng = np.random.default_rng(7)
table = tmp_db.create_table(
"rebuild_index",
[{"id": i, "vector": v.tolist()} for i, v in enumerate(rng.random((512, 4)))],
)
table.create_index(
"vector", config=lancedb.index.IvfFlat(distance_type="l2", num_partitions=4)
)
original_index = table.list_indices()[0]
table.add(
[
{"id": 512 + i, "vector": v.tolist()}
for i, v in enumerate(rng.random((64, 4)) + 5)
]
)
assert table.index_stats("vector_idx").num_unindexed_rows == 64
# --8<-- [start:storage_rebuild_vector_index]
from lancedb.index import IvfFlat
table.create_index(
"vector",
config=IvfFlat(distance_type="l2", num_partitions=4),
replace=True,
)
# --8<-- [end:storage_rebuild_vector_index]
indices = table.list_indices()
assert len(indices) == 1
assert indices[0].name == original_index.name
assert indices[0].index_uuid != original_index.index_uuid
stats = table.index_stats("vector_idx")
assert stats.num_indexed_rows == 576
assert stats.num_unindexed_rows == 0
def test_local_shallow_clone_pins_source_version(tmp_path):
db = lancedb.connect(str(tmp_path))
source = db.create_table("source", [{"id": 1}])
clone = db.clone_table("snapshot", source_uri=str(tmp_path / "source.lance"))
source.add([{"id": 2}])
assert source.count_rows() == 2
assert clone.to_arrow().to_pylist() == [{"id": 1}]
def _reader():
return pa.RecordBatchReader.from_batches(
pa.schema([("id", pa.int64())]),
[pa.record_batch([[1, 2]], names=["id"]), pa.record_batch([[3]], names=["id"])],
)
@pytest.mark.parametrize("parallelism", [1, 2])
def test_streaming_write_parallelism(tmp_db, parallelism):
table = tmp_db.create_table("streaming", [{"id": 0}])
progress = []
table.add(_reader(), write_parallelism=parallelism, progress=progress.append)
assert sorted(table.to_arrow()["id"].to_pylist()) == [0, 1, 2, 3]
assert progress[-1]["done"]
assert progress[-1]["output_rows"] == 3
assert {p["total_tasks"] for p in progress} == {parallelism}
@pytest.mark.asyncio
@pytest.mark.parametrize("parallelism", [1, 2])
async def test_async_streaming_write_parallelism(mem_db_async, parallelism):
table = await mem_db_async.create_table("streaming", [{"id": 0}])
progress = []
await table.add(_reader(), write_parallelism=parallelism, progress=progress.append)
data = await table.to_arrow()
assert sorted(data["id"].to_pylist()) == [0, 1, 2, 3]
assert progress[-1]["done"]
assert progress[-1]["output_rows"] == 3
assert {p["total_tasks"] for p in progress} == {parallelism}
docs/web-tests/py/test_embedding.py
+141
View File
@@ -0,0 +1,141 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import lancedb
# --8<-- [start:imports]
from lancedb.pydantic import LanceModel, Vector
from lancedb.embeddings import get_registry
# --8<-- [end:imports]
import pytest
@pytest.mark.skip(reason="OpenAI is not available in the test environment")
def test_create_embedding_function():
# --8<-- [start:create_embedding_function]
func = get_registry().get("openai").create(
name="text-embedding-3-small",
max_retries=7,
)
# --8<-- [end:create_embedding_function]
assert func is not None
@pytest.mark.skip(reason="OpenAI is not available in the test environment")
def test_embeddings_openai():
# --8<-- [start:openai_embeddings]
db = lancedb.connect("/tmp/db")
func = get_registry().get("openai").create(name="text-embedding-ada-002")
class Words(LanceModel):
text: str = func.SourceField()
vector: Vector(func.ndims()) = func.VectorField()
table = db.create_table("words", schema=Words, mode="overwrite")
table.add([{"text": "hello world"}, {"text": "goodbye world"}])
query = "greetings"
actual = table.search(query).limit(1).to_pydantic(Words)[0]
print(actual.text)
# --8<-- [end:openai_embeddings]
@pytest.mark.skip(reason="OpenAI is not available in the test environment")
@pytest.mark.asyncio
async def test_embeddings_openai_async():
uri = "memory://"
# --8<-- [start:async_openai_embeddings]
db = await lancedb.connect_async(uri)
func = get_registry().get("openai").create(name="text-embedding-ada-002")
class Words(LanceModel):
text: str = func.SourceField()
vector: Vector(func.ndims()) = func.VectorField()
table = await db.create_table("words", schema=Words, mode="overwrite")
await table.add([{"text": "hello world"}, {"text": "goodbye world"}])
query = "greetings"
actual = await (await table.search(query)).limit(1).to_pydantic(Words)[0]
print(actual.text)
# --8<-- [end:async_openai_embeddings]
@pytest.mark.skip(reason="OpenAI is not available in the test environment")
def test_embeddings_manual_query():
# --8<-- [start:manual_query_embeddings]
db = lancedb.connect("/tmp/db")
func = get_registry().get("openai").create(name="text-embedding-ada-002")
class Words(LanceModel):
text: str = func.SourceField()
vector: Vector(func.ndims()) = func.VectorField()
table = db.create_table("words", schema=Words, mode="overwrite")
table.add([{"text": "hello world"}, {"text": "goodbye world"}])
query_vector = func.generate_embeddings(["greetings"])[0]
# --8<-- [start:manual_query_search]
# query_vector is assumed to already be generated by your embedding function
actual = table.search(query_vector).limit(1).to_pydantic(Words)[0]
print(actual.text)
# --8<-- [end:manual_query_search]
# --8<-- [end:manual_query_embeddings]
def test_custom_embedding_function():
# --8<-- [start:embedding_function]
from functools import cached_property
from lancedb.embeddings import TextEmbeddingFunction, register
class MyEmbeddingModel:
def __init__(self, model_name: str):
self.model_name = model_name
def encode(self, texts: list[str]) -> list[list[float]]:
return [[1.0, 2.0, 3.0] for _ in texts]
@register("my-embedder")
class MyTextEmbedder(TextEmbeddingFunction):
model_name: str = "my-model"
def generate_embeddings(self, texts: list[str]) -> list[list[float]]:
# Your embedding logic here
return self._model.encode(texts)
def ndims(self) -> int:
# Return the dimensionality of the embeddings
return len(self.generate_embeddings(["test"])[0])
@cached_property
def _model(self) -> MyEmbeddingModel:
# Initialize your model once
return MyEmbeddingModel(self.model_name)
# --8<-- [end:embedding_function]
def test_embeddings_secret():
# --8<-- [start:register_secret]
registry = get_registry()
registry.set_var("api_key", "sk-...")
func = registry.get("openai").create(api_key="$var:api_key")
# --8<-- [end:register_secret]
try:
import torch
except ImportError:
pytest.skip("torch not installed")
# --8<-- [start:register_device]
import torch
registry = get_registry()
if torch.cuda.is_available():
registry.set_var("device", "cuda")
func = registry.get("huggingface").create(device="$var:device:cpu")
# --8<-- [end:register_device]
assert func.device == "cuda" if torch.cuda.is_available() else "cpu"
docs/web-tests/py/test_function.py
+242
View File
@@ -0,0 +1,242 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""Examples for docs/web/management/function.mdx.
Registering a Function and filling its column need a LanceDB Enterprise
deployment, so `test_function_enterprise` runs only when the connection
variables the Enterprise examples already use are set: LANCEDB_URI,
LANCEDB_API_KEY and LANCEDB_HOST_OVERRIDE, plus LANCEDB_REGION if the
deployment needs one. The walkthrough calls its table `function_demo` and its
Function `double`; the test creates both under names unique to its run, never
replaces an existing object, and drops only what it created.
The other tests run locally and check the client side of the same code: what
`@udf` sends for registration, how a registered version binds a column, and
where a local connection stops. They do not run a Function remotely.
"""
import json
import os
import sys
import uuid
from pathlib import Path
import lancedb
import pyarrow as pa
import pytest
from lancedb import col
from lancedb.functions import FunctionApplication, FunctionVersion
# --8<-- [start:function_define]
from lancedb import udf
@udf
def double(value: float) -> float:
return value * 2
# --8<-- [end:function_define]
# The definition the walkthrough registers. The Enterprise test registers a copy
# of it under a name that only its run uses.
WALKTHROUGH_DOUBLE = double
def describe(value: float):
return {"half": value / 2, "note": None}
# A Function version as the service returns it after registration: the SDK's
# shared golden reply, with the name and signature of the definition above.
SERVICE_REPLY = (
Path(__file__).parents[3]
/ "rust"
/ "lancedb"
/ "tests"
/ "fixtures"
/ "first_class_functions"
/ "v1"
/ "remote_function_job.json"
)
def registered_version(definition) -> FunctionVersion:
request = definition.registration_request
reply = json.loads(SERVICE_REPLY.read_text())["result"]
reply["name"] = request.name
reply["signature"] = json.loads(request.signature.to_canonical_json())
return FunctionVersion.from_json(json.dumps(reply))
def require_env(var_name: str) -> str:
"""Skip the test unless the required environment variable is provided."""
value = os.environ.get(var_name)
if not value:
pytest.skip(f"Set {var_name} to run this Enterprise example")
return value
def test_function_runs_locally_as_python():
# --8<-- [start:function_local_call]
print(double(1.5))
# 3.0
# --8<-- [end:function_local_call]
assert double(1.5) == 3.0
def test_function_registration_request():
request = double.registration_request
assert request.name == "double"
assert [
(parameter.name, parameter.arrow_type, parameter.nullable)
for parameter in request.signature.inputs
] == [("value", "float64", False)]
output = request.signature.output
assert (output.kind, output.arrow_type, output.nullable) == (
"scalar",
"float64",
False,
)
version = request.runtime.python_version
assert version == f"{sys.version_info.major}.{sys.version_info.minor}"
assert request.runtime.environment.packages == ()
def test_function_output_nullability():
inputs = pa.schema([pa.field("value", pa.float64(), nullable=False)])
# A named-struct output may declare nullable fields.
fields = [
pa.field("half", pa.float64(), nullable=False),
pa.field("note", pa.string(), nullable=True),
]
struct = udf(input_schema=inputs, output_schema=pa.schema(fields))(describe)
output = struct.registration_request.signature.output
assert output.kind == "named_struct"
assert [(field.name, field.nullable) for field in output.fields] == [
("half", False),
("note", True),
]
# A scalar output, and a struct declared as one nullable field, may not be.
for nullable in (
pa.field("result", pa.float64(), nullable=True),
pa.field("result", pa.struct(fields), nullable=True),
):
with pytest.raises(ValueError, match="non-nullable"):
udf(input_schema=inputs, output_schema=nullable)(describe)
def test_function_registration_is_remote(tmp_path):
db = lancedb.connect(tmp_path)
with pytest.raises(NotImplementedError):
db.create_function(double)
def test_function_version_binds_a_column(tmp_path):
version = registered_version(double)
application = version(value=col("value"))
assert isinstance(application, FunctionApplication)
assert application.function.name == "double"
assert application.function.version == version.version
assert [
(binding.parameter, binding.kind, binding.value["path"])
for binding in application.inputs
] == [("value", "column", "value")]
with pytest.raises(TypeError, match="direct col"):
version(value=1.5)
with pytest.raises(TypeError, match="missing"):
version(x=col("value"))
# The binding is accepted by add_columns, which a local table refuses:
# a Function column needs LanceDB Cloud or Enterprise.
table = lancedb.connect(tmp_path).create_table("t", [{"value": 1.0}])
with pytest.raises(NotImplementedError, match="Cloud and Enterprise"):
table.add_columns({"doubled": application})
def test_function_enterprise(monkeypatch):
require_env("LANCEDB_URI")
require_env("LANCEDB_API_KEY")
require_env("LANCEDB_HOST_OVERRIDE")
# --8<-- [start:function_connect]
import os
import lancedb
db = lancedb.connect(
uri=os.environ["LANCEDB_URI"], # db://your-database
api_key=os.environ["LANCEDB_API_KEY"],
region=os.environ.get("LANCEDB_REGION", "us-east-1"),
host_override=os.environ["LANCEDB_HOST_OVERRIDE"],
)
# --8<-- [end:function_connect]
# In a shared database `function_demo` or `double` may already exist, so
# this run creates both under names only it uses, and drops only those.
run = uuid.uuid4().hex[:12]
table_name = f"function_demo_{run}"
double = udf(name=f"double_{run}")(WALKTHROUGH_DOUBLE.__wrapped__)
create_table = db.create_table
def create_run_table(name, *args, **kwargs):
assert name == "function_demo"
return create_table(table_name, *args, **kwargs)
monkeypatch.setattr(db, "create_table", create_run_table)
table = None
version = None
try:
# --8<-- [start:function_create_table]
import pyarrow as pa
schema = pa.schema([pa.field("value", pa.float64(), nullable=False)])
table = db.create_table(
"function_demo",
[{"value": 1.0}, {"value": 2.0}, {"value": 3.0}],
schema=schema,
)
# --8<-- [end:function_create_table]
# --8<-- [start:function_register]
job = db.create_function_async(double)
version = job.wait()
# --8<-- [end:function_register]
assert version.name == f"double_{run}"
# --8<-- [start:function_apply]
from lancedb import col
table.add_columns({"doubled": version(value=col("value"))})
# --8<-- [end:function_apply]
# --8<-- [start:function_refresh]
refresh = table.refresh_column_async("doubled")
result = refresh.wait()
print(result.rows_assigned)
# 3
# --8<-- [end:function_refresh]
assert result.rows_assigned == 3
assert result.rows_failed == 0
# --8<-- [start:function_inspect]
rows = table.search().select(["value", "doubled"]).to_list()
print(sorted((row["value"], row["doubled"]) for row in rows))
# [(1.0, 2.0), (2.0, 4.0), (3.0, 6.0)]
# --8<-- [end:function_inspect]
assert sorted((row["value"], row["doubled"]) for row in rows) == [
(1.0, 2.0),
(2.0, 4.0),
(3.0, 6.0),
]
finally:
if table is not None:
db.drop_table(table_name)
if version is not None:
db.drop_function(version.name, version=version.version)
docs/web-tests/py/test_indexing.py
+757
View File
@@ -0,0 +1,757 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import random
import string
import uuid
import numpy as np
import pyarrow as pa
import pytest
def _make_vector_rows(count: int, dim: int, column: str = "vector"):
rows = [
{column: np.random.random(dim).astype(np.float32).tolist(), "id": i}
for i in range(count)
]
assert len(rows) == count
assert all(len(row[column]) == dim for row in rows)
return rows
def test_vector_index_configure_ivf(tmp_db):
table = tmp_db.create_table(
"vector_index_configure_ivf",
_make_vector_rows(512, 4),
mode="overwrite",
)
# --8<-- [start:vector_index_configure_ivf]
table.create_index(metric="l2", num_partitions=16, num_sub_vectors=4)
# --8<-- [end:vector_index_configure_ivf]
assert table.list_indices()
def test_vector_index_setup(tmp_db):
tmp_db.create_table(
"vector-index-tbl",
_make_vector_rows(8, 4),
mode="overwrite",
)
db = tmp_db
# --8<-- [start:vector_index_setup]
table_name = "vector-index-tbl"
table = db.open_table(table_name)
# --8<-- [end:vector_index_setup]
assert table.name == table_name
def test_vector_index_build_ivf(tmp_db):
table = tmp_db.create_table(
"vector-index-build-ivf",
_make_vector_rows(512, 4, column="keywords_embeddings"),
mode="overwrite",
)
db = tmp_db
# --8<-- [start:vector_index_build_ivf]
table_name = "vector-index-build-ivf"
table = db.open_table(table_name)
table.create_index(
metric="cosine",
vector_column_name="keywords_embeddings",
)
# --8<-- [end:vector_index_build_ivf]
assert table.list_indices()
def test_vector_index_nested_field(tmp_db):
dim = 2
schema = pa.schema(
[
pa.field("id", pa.int32()),
pa.field(
"image",
pa.struct([pa.field("embedding", pa.list_(pa.float32(), dim))]),
),
]
)
data = [
{
"id": i,
"image": {"embedding": np.random.random(dim).astype(np.float32).tolist()},
}
for i in range(512)
]
table = tmp_db.create_table(
"vector_index_nested_field", data=data, schema=schema, mode="overwrite"
)
# --8<-- [start:vector_index_nested_field]
# The vector column `embedding` is nested inside the `image` struct.
# Pass its full dotted path as `vector_column_name`; the same path is used
# at query time and is what `list_indices()` reports under `columns`.
table.create_index(
vector_column_name="image.embedding",
num_partitions=1,
num_sub_vectors=1,
name="image_embedding_idx",
)
results = (
table.search([0.0, 1.0], vector_column_name="image.embedding")
.limit(1)
.to_list()
)
# --8<-- [end:vector_index_nested_field]
assert table.index_stats("image_embedding_idx")
assert len(results) == 1
@pytest.mark.asyncio
async def test_vector_index_async_config(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
# --8<-- [start:vector_index_async_config]
import lancedb
import numpy as np
from lancedb.index import IvfPq
async def main():
data = [
{"id": i, "vector": np.random.random(8).astype(np.float32).tolist()}
for i in range(512)
]
db = await lancedb.connect_async("ex_lancedb")
table = await db.create_table(
"vector_index_async", data=data, mode="overwrite"
)
await table.create_index(
"vector",
config=IvfPq(
distance_type="cosine",
num_partitions=16,
num_sub_vectors=4,
),
)
return await table.list_indices()
# --8<-- [end:vector_index_async_config]
assert await main()
def test_vector_index_query_ivf(tmp_db):
dim = 1536
data = [
{"id": i, "keywords_embeddings": np.random.random(dim).tolist()}
for i in range(512)
]
table = tmp_db.create_table("vector_index_query_ivf", data, mode="overwrite")
table.create_index(
metric="cosine",
vector_column_name="keywords_embeddings",
)
# --8<-- [start:vector_index_query_ivf]
tbl = table
tbl.search(np.random.random((1536))).limit(2).nprobes(20).refine_factor(
10
).to_pandas()
# --8<-- [end:vector_index_query_ivf]
df = (
table.search(np.random.random((1536)))
.limit(2)
.nprobes(20)
.refine_factor(10)
.to_pandas()
)
assert len(df) == 2
def test_vector_index_nprobes(tmp_db):
dim = 128
data = [
{"id": i, "keywords_embeddings": np.random.random(dim).tolist()}
for i in range(512)
]
table = tmp_db.create_table("vector_index_nprobes", data, mode="overwrite")
table.create_index(
metric="cosine",
vector_column_name="keywords_embeddings",
)
# --8<-- [start:vector_index_nprobes]
# Always scan 10 partitions; scan up to 50 only if the initial pass
# returns fewer than `limit` results (common with narrow filters).
(
table.search(np.random.random(128))
.minimum_nprobes(10)
.maximum_nprobes(50)
.where("id > 100")
.limit(5)
.to_pandas()
)
# --8<-- [end:vector_index_nprobes]
def test_vector_index_distance_range(tmp_db):
dim = 128
data = [
{"id": i, "keywords_embeddings": np.random.random(dim).tolist()}
for i in range(256)
]
table = tmp_db.create_table("vector_index_distance_range", data, mode="overwrite")
table.create_index(
metric="cosine",
vector_column_name="keywords_embeddings",
)
# --8<-- [start:vector_index_distance_range]
# Only return results whose distance falls within [0.0, 0.5).
# Useful for near-duplicate detection or thresholded similarity search.
(
table.search(np.random.random(128))
.distance_range(lower_bound=0.0, upper_bound=0.5)
.limit(10)
.to_pandas()
)
# --8<-- [end:vector_index_distance_range]
def test_vector_index_bypass_recall(tmp_db):
dim = 128
data = [
{"id": i, "keywords_embeddings": np.random.random(dim).tolist()}
for i in range(256)
]
table = tmp_db.create_table("vector_index_bypass_recall", data, mode="overwrite")
table.create_index(
metric="cosine",
vector_column_name="keywords_embeddings",
)
# --8<-- [start:vector_index_bypass_recall]
query = np.random.random(128)
k = 10
# Ground truth: flat (exhaustive) scan, ignoring the ANN index.
truth = set(table.search(query).bypass_vector_index().limit(k).to_pandas()["id"])
# ANN results with the current nprobes setting.
ann = set(table.search(query).nprobes(20).limit(k).to_pandas()["id"])
recall_at_k = len(truth & ann) / k
# --8<-- [end:vector_index_bypass_recall]
assert 0.0 <= recall_at_k <= 1.0
def test_vector_index_custom_name(tmp_db):
table = tmp_db.create_table(
"vector_index_custom_name",
_make_vector_rows(512, 8, column="keywords_embeddings"),
mode="overwrite",
)
# --8<-- [start:vector_index_custom_name]
# Override the default `{column}_idx` convention by passing `name=...`.
table.create_index(
metric="cosine",
vector_column_name="keywords_embeddings",
name="my_custom_index",
)
table.wait_for_index(["my_custom_index"])
print(table.index_stats("my_custom_index"))
# --8<-- [end:vector_index_custom_name]
assert table.index_stats("my_custom_index")
def test_vector_index_hnsw(tmp_db):
table = tmp_db.create_table(
"vector_index_hnsw",
_make_vector_rows(64, 16),
mode="overwrite",
)
# --8<-- [start:vector_index_build_hnsw]
table.create_index(index_type="IVF_HNSW_SQ")
# --8<-- [end:vector_index_build_hnsw]
# --8<-- [start:vector_index_query_hnsw]
tbl = table
tbl.search(np.random.random((16))).limit(2).to_pandas()
# --8<-- [end:vector_index_query_hnsw]
df = table.search(np.random.random((16))).limit(2).to_pandas()
assert len(df) == 2
def test_vector_index_binary(tmp_db):
table_name = "hamming-index-tbl"
ndim = 256
schema = pa.schema(
[
pa.field("id", pa.int64()),
pa.field("vector", pa.list_(pa.uint8(), ndim // 8)),
]
)
# --8<-- [start:vector_index_binary_schema]
table = tmp_db.create_table(table_name, schema=schema, mode="overwrite")
# --8<-- [end:vector_index_binary_schema]
data = []
for i in range(64):
vector = np.random.randint(0, 2, size=ndim)
vector = np.packbits(vector)
data.append({"id": i, "vector": vector})
# --8<-- [start:vector_index_binary_add_data]
table.add(data)
# --8<-- [end:vector_index_binary_add_data]
# --8<-- [start:vector_index_binary_build_index]
table.create_index(
metric="hamming",
vector_column_name="vector",
index_type="IVF_FLAT",
)
# --8<-- [end:vector_index_binary_build_index]
# --8<-- [start:vector_index_binary_search]
query = np.random.randint(0, 2, size=ndim)
query = np.packbits(query)
df = table.search(query).metric("hamming").limit(10).to_pandas()
df.vector = df.vector.apply(np.unpackbits)
# --8<-- [end:vector_index_binary_search]
assert not df.empty
def test_quantization_custom_params(tmp_db):
table = tmp_db.create_table(
"quantization-custom-params",
_make_vector_rows(256, 64),
mode="overwrite",
)
# --8<-- [start:quantization_custom_params]
table.create_index(
index_type="IVF_RQ",
num_bits=2,
max_iterations=100,
sample_rate=512,
)
# --8<-- [end:quantization_custom_params]
assert table.list_indices()
def test_vector_index_check_status(tmp_db):
table = tmp_db.create_table(
"vector_index_check_status",
_make_vector_rows(512, 8, column="keywords_embeddings"),
mode="overwrite",
)
table.create_index(
metric="cosine",
vector_column_name="keywords_embeddings",
)
# --8<-- [start:vector_index_check_status]
index_name = "keywords_embeddings_idx"
table.wait_for_index([index_name])
print(table.index_stats(index_name))
# --8<-- [end:vector_index_check_status]
assert table.index_stats(index_name)
def test_scalar_index_build(tmp_db):
table = tmp_db.create_table(
"scalar_index_build",
[
{"book_id": 1, "publisher": "A", "vector": [0.1, 0.2]},
{"book_id": 2, "publisher": "B", "vector": [0.2, 0.3]},
],
mode="overwrite",
)
db = tmp_db
# --8<-- [start:scalar_index_build]
tbl = db.open_table("scalar_index_build")
tbl.create_scalar_index("book_id")
tbl.create_scalar_index("publisher", index_type="BITMAP")
# --8<-- [end:scalar_index_build]
assert tbl.list_indices()
def test_scalar_index_wait(tmp_db):
table = tmp_db.create_table(
"scalar_index_wait",
[{"label": "fiction"}],
mode="overwrite",
)
table.create_scalar_index("label")
# --8<-- [start:scalar_index_wait]
index_name = "label_idx"
table.wait_for_index([index_name])
# --8<-- [end:scalar_index_wait]
assert table.list_indices()
def test_scalar_index_optimize(tmp_db):
table = tmp_db.create_table(
"scalar_index_optimize",
[{"vector": [7.0, 8.0], "book_id": 3}],
mode="overwrite",
)
# --8<-- [start:scalar_index_optimize]
table.add([{"vector": [7, 8], "book_id": 4}])
table.optimize()
# --8<-- [end:scalar_index_optimize]
result = table.search().where("book_id = 4").limit(10).to_pandas()
assert len(result) == 1
def test_scalar_index_filter(tmp_db):
table = tmp_db.create_table(
"books",
[
{"vector": [1.1, 1.2], "book_id": 1},
{"vector": [2.1, 2.2], "book_id": 2},
],
mode="overwrite",
)
db = tmp_db
# --8<-- [start:scalar_index_filter]
table = db.open_table("books")
result = table.search().where("book_id = 2").limit(10).to_pandas()
# --8<-- [end:scalar_index_filter]
assert len(result) == 1
def test_scalar_index_prefilter(tmp_db):
table = tmp_db.create_table(
"book_with_embeddings",
[
{"vector": [1.2, 1.3], "book_id": 1},
{"vector": [4.2, 4.3], "book_id": 2},
],
mode="overwrite",
)
db = tmp_db
# --8<-- [start:scalar_index_prefilter]
table = db.open_table("book_with_embeddings")
table.search([1.2] * 2).where("book_id != 3").limit(10).to_pandas()
# --8<-- [end:scalar_index_prefilter]
result = table.search([1.2] * 2).where("book_id != 3").limit(10).to_pandas()
assert len(result) == 2
def test_scalar_index_uuid(tmp_db):
# --8<-- [start:scalar_index_uuid_type]
import pyarrow as pa
# --8<-- [end:scalar_index_uuid_type]
# --8<-- [start:scalar_index_uuid_data]
def generate_random_names():
base_names = ["Alice", "Bob", "Carla", "David", "Eve", "Frank", "Grace"]
letter = random.choice(string.ascii_uppercase)
return f"{random.choice(base_names)} {letter}."
def generate_uuids(num_items):
return [uuid.uuid4().bytes for _ in range(num_items)]
# Generate some UUIDs and random names
n = 7
uuids = generate_uuids(n)
names = [generate_random_names() for _ in range(n)]
# --8<-- [end:scalar_index_uuid_data]
db = tmp_db
# --8<-- [start:scalar_index_uuid_table]
table_name = "index-on-uuid"
uuid_array = pa.array(uuids, pa.uuid())
name_array = pa.array(names, pa.string())
schema = pa.schema(
[
pa.field("id", pa.uuid()),
pa.field("name", pa.string()),
]
)
data_table = pa.Table.from_arrays([uuid_array, name_array], schema=schema)
table = db.create_table(table_name, data=data_table, mode="overwrite")
# --8<-- [end:scalar_index_uuid_table]
# --8<-- [start:scalar_index_uuid_wait]
index_name = "id_idx"
table.create_scalar_index("id")
table.wait_for_index([index_name])
# --8<-- [end:scalar_index_uuid_wait]
# --8<-- [start:scalar_index_uuid_upsert]
new_users = [
{"id": uuid.uuid4().bytes, "name": "Hannah D."},
{"id": uuid.uuid4().bytes, "name": "Ian B."},
]
# Insert or update using the UUID index
table.merge_insert(
"id"
).when_matched_update_all().when_not_matched_insert_all().execute(new_users)
# --8<-- [end:scalar_index_uuid_upsert]
assert table.list_indices()
result = table.search().limit(100).to_pandas()
assert len(result) == n + len(new_users)
@pytest.mark.asyncio
async def test_scalar_index_nested_fields(mem_db_async):
db = mem_db_async
# --8<-- [start:scalar_index_nested_fields]
import pyarrow as pa
from lancedb.index import BTree
metadata_type = pa.struct(
[
pa.field("user_id", pa.int32()),
pa.field("user.id", pa.int32()),
]
)
data = pa.Table.from_arrays(
[
pa.array([1, 2, 3], type=pa.int32()),
pa.array(
[
{"user_id": 10, "user.id": 100},
{"user_id": 20, "user.id": 200},
{"user_id": 30, "user.id": 300},
],
type=metadata_type,
),
],
names=["user_id", "metadata"],
)
table = await db.create_table("nested_scalar_index", data)
# Index a nested struct field.
await table.create_index(
"metadata.user_id", config=BTree(), name="nested_user_id_idx"
)
# Escape literal dots inside a segment with backticks.
await table.create_index(
"metadata.`user.id`", config=BTree(), name="escaped_user_id_idx"
)
# `columns` is returned as the canonical path you passed in.
for index in await table.list_indices():
print(index.name, index.columns)
# nested_user_id_idx ['metadata.user_id']
# escaped_user_id_idx ['metadata.`user.id`']
# --8<-- [end:scalar_index_nested_fields]
index_columns = {index.name: index.columns for index in await table.list_indices()}
assert index_columns["nested_user_id_idx"] == ["metadata.user_id"]
assert index_columns["escaped_user_id_idx"] == ["metadata.`user.id`"]
def test_fts_index_create(tmp_db):
table = tmp_db.create_table(
"fts-index-create",
[{"text": "hello world", "vector": [0.1, 0.2]}],
mode="overwrite",
)
db = tmp_db
# --8<-- [start:fts_index_create]
table_name = "fts-index-create"
table = db.open_table(table_name)
table.create_fts_index("text")
# --8<-- [end:fts_index_create]
assert table.list_indices()
def test_fts_index_wait(tmp_db):
table = tmp_db.create_table(
"fts-index-wait",
[{"text": "full text search"}],
mode="overwrite",
)
db = tmp_db
# --8<-- [start:fts_index_wait]
table_name = "fts-index-wait"
table = db.open_table(table_name)
table.create_fts_index("text")
index_name = "text_idx"
table.wait_for_index([index_name])
# --8<-- [end:fts_index_wait]
assert table.list_indices()
def test_fts_index_nested_field(tmp_db):
nested_schema = pa.struct([
pa.field("text", pa.string()),
pa.field("count", pa.int32()),
])
schema = pa.schema([
pa.field("id", pa.int64()),
pa.field("payload", nested_schema),
])
tmp_db.create_table(
"fts-index-nested",
pa.table(
{
"id": pa.array([1, 2], pa.int64()),
"payload": pa.array(
[
{"text": "Frodo was a happy puppy", "count": 1},
{"text": "puppy runs through the meadow", "count": 2},
],
type=nested_schema,
),
},
schema=schema,
),
mode="overwrite",
)
db = tmp_db
# --8<-- [start:fts_index_nested]
from lancedb.query import MatchQuery, PhraseQuery
table = db.open_table("fts-index-nested")
# Index a text leaf inside a struct column using a dotted path.
table.create_fts_index("payload.text", with_position=True)
# The same dotted path works in MatchQuery and PhraseQuery.
matches = (
table.search(MatchQuery("puppy", "payload.text")).limit(5).to_list()
)
phrases = (
table.search(PhraseQuery("puppy runs", "payload.text"))
.limit(5)
.to_list()
)
# --8<-- [end:fts_index_nested]
assert len(matches) > 0
assert all("puppy" in row["payload"]["text"] for row in matches)
assert len(phrases) > 0
assert all("puppy runs" in row["payload"]["text"] for row in phrases)
@pytest.mark.asyncio
async def test_fts_index_async(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
# --8<-- [start:fts_index_async]
import asyncio
import lancedb
import polars as pl
from lancedb.index import FTS
data = pl.DataFrame(
{
"id": [1, 2],
"text": [
"His first language is spanish",
"Her first language is english",
],
}
)
async def main(data: pl.DataFrame):
uri = "ex_lancedb"
db = await lancedb.connect_async(uri)
tbl = await db.create_table("my_text", data=data, mode="overwrite")
await tbl.create_index("text", config=FTS(language="English"))
response = await tbl.search("spanish", query_type="fts")
result = await response.limit(1).to_polars()
print(result)
return result
if __name__ == "__main__":
asyncio.run(main(data))
# --8<-- [end:fts_index_async]
result = await main(data)
assert result.height == 1
def test_gpu_index_snippets(tmp_db, monkeypatch):
table = tmp_db.create_table(
"gpu_index",
_make_vector_rows(32, 8),
mode="overwrite",
)
calls = []
def fake_create_index(*args, **kwargs):
calls.append(kwargs)
return None
monkeypatch.setattr(table, "create_index", fake_create_index)
# --8<-- [start:gpu_index_cuda]
table.create_index(
num_partitions=256,
num_sub_vectors=96,
accelerator="cuda",
)
# --8<-- [end:gpu_index_cuda]
# --8<-- [start:gpu_index_mps]
table.create_index(
num_partitions=256,
num_sub_vectors=96,
accelerator="mps",
)
# --8<-- [end:gpu_index_mps]
assert calls[0]["accelerator"] == "cuda"
assert calls[1]["accelerator"] == "mps"
def test_reindexing_incremental(tmp_db):
table = tmp_db.create_table(
"reindexing_incremental",
[{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}],
mode="overwrite",
)
db = tmp_db
# --8<-- [start:reindexing_incremental]
table = db.open_table("reindexing_incremental")
table.add([{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}])
table.optimize()
# --8<-- [end:reindexing_incremental]
result = table.search().limit(10).to_pandas()
assert len(result) == 2
docs/web-tests/py/test_integrations.py
+1767
View File
File diff suppressed because it is too large Load Diff
docs/web-tests/py/test_materialized_view.py
+69
View File
@@ -0,0 +1,69 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""Examples for docs/web/feature-engineering/materialized-views.mdx."""
def test_materialized_view_create_and_refresh(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
# --8<-- [start:materialized_view_create]
import lancedb
db = lancedb.connect("./.lancedb")
table = db.create_table(
"people", [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}]
)
view = db.create_materialized_view(
"adults",
"people",
select=["name", ("shout", "upper(name)")],
where="age >= 18",
)
print(view.table.count_rows())
# 1
# --8<-- [end:materialized_view_create]
assert view.table.count_rows() == 1
# --8<-- [start:materialized_view_refresh]
table.add([{"name": "bea", "age": 41}])
result = view.refresh()
print(result.rows_written)
# 2
# --8<-- [end:materialized_view_refresh]
assert result.mode == "rebuild"
assert result.rows_written == 2
assert sorted(row["shout"] for row in view.table.to_arrow().to_pylist()) == [
"ADA",
"BEA",
]
def test_materialized_view_with_no_data(tmp_path):
import lancedb
db = lancedb.connect(tmp_path)
db.create_table("people", [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}])
view = db.create_materialized_view(
"adults", "people", where="age >= 18", with_no_data=True
)
assert view.table.count_rows() == 0
assert view.refresh().rows_written == 1
assert view.table.count_rows() == 1
def test_materialized_view_refresh_to_source_version(tmp_path):
import lancedb
db = lancedb.connect(tmp_path)
table = db.create_table("people", [{"name": "ada", "age": 36}])
view = db.create_materialized_view("adults", "people", where="age >= 18")
first = table.version
table.add([{"name": "bea", "age": 41}])
assert view.refresh().rows_written == 2
result = view.refresh(source_version=first)
assert (result.source_version, result.rows_written) == (first, 1)
assert view.table.count_rows() == 1
docs/web-tests/py/test_multimodal.py
+232
View File
@@ -0,0 +1,232 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import pytest
try:
import lancedb
import numpy as np
import pyarrow as pa
import io
from PIL import Image
except ImportError:
pass
# --8<-- [start:multimodal_imports]
import lancedb
import pyarrow as pa
import pandas as pd
import numpy as np
import io
from PIL import Image
# --8<-- [end:multimodal_imports]
def test_multimodal_ingestion(db_path_factory):
# Ensure dependencies are available
pytest.importorskip("PIL")
pytest.importorskip("lancedb")
pytest.importorskip("numpy")
# --8<-- [start:create_dummy_data]
# Create some dummy images
def create_dummy_image(color):
img = Image.new('RGB', (100, 100), color=color)
buf = io.BytesIO()
img.save(buf, format='PNG')
return buf.getvalue()
# Create dataset with metadata, vectors, and image blobs
data = [
{
"id": 1,
"filename": "red_square.png",
"vector": np.random.rand(128).astype(np.float32),
"image_blob": create_dummy_image('red'),
"label": "red"
},
{
"id": 2,
"filename": "blue_square.png",
"vector": np.random.rand(128).astype(np.float32),
"image_blob": create_dummy_image('blue'),
"label": "blue"
}
]
# --8<-- [end:create_dummy_data]
# --8<-- [start:define_schema]
# Define schema explicitly to ensure image_blob is treated as binary
schema = pa.schema([
pa.field("id", pa.int32()),
pa.field("filename", pa.string()),
pa.field("vector", pa.list_(pa.float32(), 128)),
pa.field("image_blob", pa.binary()), # Important: Use pa.binary() for blobs
pa.field("label", pa.string())
])
# --8<-- [end:define_schema]
db_uri = db_path_factory("multimodal_db")
db = lancedb.connect(db_uri)
# --8<-- [start:ingest_data]
tbl = db.create_table("images", data=data, schema=schema, mode="overwrite")
# --8<-- [end:ingest_data]
assert len(tbl) == 2
# --8<-- [start:search_data]
# Search for similar images
query_vector = np.random.rand(128).astype(np.float32)
results = tbl.search(query_vector).limit(1).to_pandas()
# --8<-- [end:search_data]
# --8<-- [start:process_results]
# Convert back to PIL Image
for _, row in results.iterrows():
image_bytes = row['image_blob']
image = Image.open(io.BytesIO(image_bytes))
print(f"Retrieved image: {row['filename']}, Size: {image.size}")
# You can now use 'image' with other libraries or display it
# --8<-- [end:process_results]
assert len(results) == 1
def test_blob_api_definition(db_path_factory):
# --8<-- [start:blob_api_schema]
import pyarrow as pa
# Define schema with Blob API metadata for lazy loading
schema = pa.schema([
pa.field("id", pa.int64()),
pa.field(
"video",
pa.large_binary(),
metadata={"lance-encoding:blob": "true"} # Enable Blob API
),
])
# --8<-- [end:blob_api_schema]
# --8<-- [start:blob_api_ingest]
import lancedb
db = lancedb.connect(db_path_factory("blob_db"))
# Create sample data
data = [
{"id": 1, "video": b"fake_video_bytes_1"},
{"id": 2, "video": b"fake_video_bytes_2"}
]
# Create the table
tbl = db.create_table("videos", data=data, schema=schema)
# --8<-- [end:blob_api_ingest]
assert len(tbl) == 2
def test_blob_api_to_pandas(db_path_factory):
db = lancedb.connect(db_path_factory("blob_to_pandas_db"))
schema = pa.schema([
pa.field("id", pa.int64()),
pa.field(
"video",
pa.large_binary(),
metadata={"lance-encoding:blob": "true"},
),
])
tbl = db.create_table(
"videos",
data=[
{"id": 1, "video": b"fake_video_bytes_1"},
{"id": 2, "video": b"fake_video_bytes_2"},
],
schema=schema,
mode="overwrite",
)
# --8<-- [start:blob_api_to_pandas]
# Default: blob columns come back lazily
df_lazy = tbl.to_pandas()
# Materialize blob bytes eagerly
df_bytes = tbl.to_pandas(blob_mode="bytes")
# Return descriptors instead of payloads
df_desc = tbl.to_pandas(blob_mode="descriptions")
# Forward extra kwargs to PyArrow's to_pandas
df_typed = tbl.to_pandas(split_blocks=True, self_destruct=True)
# --8<-- [end:blob_api_to_pandas]
assert len(df_lazy) == 2
assert isinstance(df_bytes["video"].iloc[0], bytes)
assert df_bytes["video"].tolist() == [
b"fake_video_bytes_1",
b"fake_video_bytes_2",
]
assert len(df_desc) == 2
assert len(df_typed) == 2
@pytest.mark.asyncio
async def test_query_to_pandas_kwargs(db_path_factory):
schema = pa.schema([
pa.field("id", pa.int64()),
pa.field("vector", pa.list_(pa.float32(), 128)),
pa.field(
"video",
pa.large_binary(),
metadata={"lance-encoding:blob": "true"},
),
])
data = [
{
"id": i,
"vector": np.random.rand(128).astype(np.float32),
"video": f"fake_video_bytes_{i}".encode(),
}
for i in range(10)
]
db = lancedb.connect(db_path_factory("query_to_pandas_db"))
tbl = db.create_table("search_demo", data=data, schema=schema, mode="overwrite")
async_db = await lancedb.connect_async(
str(db_path_factory("query_to_pandas_async_db"))
)
tbl_async = await async_db.create_table(
"search_demo", data=data, schema=schema, mode="overwrite"
)
query_vector = np.random.rand(128).astype(np.float32)
# --8<-- [start:query_to_pandas_kwargs]
# Plain scan query: blob_mode is supported end to end
df_lazy = (
tbl.search()
.where("id = 1")
.select(["id", "video"])
.to_pandas(blob_mode="lazy")
)
# Same call shape works on async query builders
df_bytes = await (
tbl_async.query()
.where("id = 1")
.select(["id", "video"])
.to_pandas(blob_mode="bytes")
)
# Vector / FTS / hybrid queries can't materialize blob columns,
# so omit them from the projection
df_vec = (
tbl.search(query_vector)
.limit(10)
.select(["id", "vector"])
.to_pandas(split_blocks=True, self_destruct=True)
)
# --8<-- [end:query_to_pandas_kwargs]
assert len(df_lazy) == 1
assert len(df_bytes) == 1
assert df_bytes["video"].iloc[0] == b"fake_video_bytes_1"
assert len(df_vec) == 10
assert "video" not in df_vec.columns
docs/web-tests/py/test_quickstart.py
+240
View File
@@ -0,0 +1,240 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import lancedb
import pytest
def test_quickstart(db_path_factory):
uri = db_path_factory("quickstart_db")
db = lancedb.connect(uri)
# --8<-- [start:quickstart_data]
data = [
{
"id": "1",
"name": "King Arthur",
"role": "King",
"description": "Leader of Camelot and wielder of Excalibur.",
"stats": {"strength": 4, "magic": 1, "leadership": 5, "wisdom": 4},
"vector": [0.7, 0.1, 0.9, 0.7],
},
{
"id": "2",
"name": "Merlin",
"role": "Wizard",
"description": "Advisor and prophet with deep magical knowledge.",
"stats": {"strength": 2, "magic": 5, "leadership": 4, "wisdom": 5},
"vector": [0.2, 0.9, 0.4, 0.9],
},
{
"id": "3",
"name": "Sir Lancelot",
"role": "Knight",
"description": "Legendary knight known for courage and combat skill.",
"stats": {"strength": 5, "magic": 1, "leadership": 3, "wisdom": 3},
"vector": [0.9, 0.1, 0.5, 0.4],
},
]
# --8<-- [end:quickstart_data]
# --8<-- [start:quickstart_create_table]
table = db.create_table("characters", data=data, mode="overwrite")
# --8<-- [end:quickstart_create_table]
assert len(table) == 3
# Drop the table to test create without overwrite
db.drop_table("characters")
# --8<-- [start:quickstart_create_table_no_overwrite]
table = db.create_table("characters", data=data)
# --8<-- [end:quickstart_create_table_no_overwrite]
assert len(table) == 3
# --8<-- [start:quickstart_vector_search_1]
# Search for examples similar to a "wise magical advisor"
query_vector = [0.2, 0.8, 0.4, 0.9]
# Ensure you run `pip install polars` beforehand
result = (
table.search(query_vector)
.select(["name", "role", "description", "_distance"])
.limit(2)
.to_polars()
)
print(result)
# --8<-- [end:quickstart_vector_search_1]
assert result.head(1)["name"][0] == "Merlin"
# --8<-- [start:quickstart_curate_with_metadata]
curated = (
table.search(query_vector)
.where("stats.magic >= 4")
.select(["name", "role", "description", "_distance"])
.limit(2)
.to_polars()
)
print(curated)
# --8<-- [end:quickstart_curate_with_metadata]
assert curated.head(1)["name"][0] == "Merlin"
# --8<-- [start:quickstart_output_pandas]
# Ensure you run `pip install pandas` beforehand
result = table.search(query_vector).limit(2).to_pandas()
print(result)
# --8<-- [end:quickstart_output_pandas]
assert result.iloc[0]["name"] == "Merlin"
# --8<-- [start:quickstart_add_feature]
table.add_columns(
{
"power_score": "cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)"
}
)
# --8<-- [end:quickstart_add_feature]
assert "power_score" in table.schema.names
# --8<-- [start:quickstart_query_feature]
features = table.search().select(["name", "role", "power_score"]).to_polars()
print(features)
# --8<-- [end:quickstart_query_feature]
assert "power_score" in features.columns
# --8<-- [start:quickstart_multimodal_bytes]
from pathlib import Path
image_path = Path("docs/web/static/assets/images/quickstart/sir-lancelot.jpg")
image_bytes = image_path.read_bytes()
multimodal_table = db.create_table(
"character_images",
data=[
{
"id": "lancelot",
"description": "Portrait of Sir Lancelot",
"image": image_bytes,
"vector": [0.9, 0.1, 0.5, 0.4],
}
],
mode="overwrite",
)
# --8<-- [end:quickstart_multimodal_bytes]
assert len(multimodal_table) == 1
# --8<-- [start:quickstart_open_table]
table = db.open_table("characters")
# --8<-- [end:quickstart_open_table]
# --8<-- [start:quickstart_add_data]
more_data = [
{
"id": "4",
"name": "Morgana",
"role": "Sorceress",
"description": "Powerful sorceress of Avalon.",
"stats": {"strength": 2, "magic": 5, "leadership": 4, "wisdom": 4},
"vector": [0.3, 0.9, 0.6, 0.8],
"power_score": 3.75,
},
]
# Add data to table
table.add(more_data)
# --8<-- [end:quickstart_add_data]
assert len(table) == 4
# --8<-- [start:quickstart_vector_search_2]
# Search for examples similar to a "powerful sorceress"
query_vector = [0.3, 0.9, 0.6, 0.8]
results = table.search(query_vector).limit(2).to_polars()
print(results)
# --8<-- [end:quickstart_vector_search_2]
assert results.head(1)["name"][0] == "Morgana"
# --8<-- [start:quickstart_computed_feature]
table.add_columns(computed={"combat_score": "(stats.magic + stats.strength) / 2.0"})
table.refresh_column("combat_score")
# --8<-- [end:quickstart_computed_feature]
scores = (
table.search().select(["combat_score"]).to_arrow()["combat_score"].to_pylist()
)
assert scores == [2.5, 3.5, 3.0, 3.5]
# --8<-- [start:quickstart_training]
import torch
from lancedb.permutation import Permutation
dataset = Permutation.identity(table).select_columns(
["power_score", "combat_score"]
)
dataloader = torch.utils.data.DataLoader(dataset, batch_size=1024, shuffle=True)
# --8<-- [end:quickstart_training]
batches = list(dataloader)
assert len(batches) == 1
assert set(batches[0]) == {"power_score", "combat_score"}
assert all(isinstance(value, torch.Tensor) for value in batches[0].values())
assert sorted(batches[0]["combat_score"].tolist()) == sorted(scores)
# --8<-- [start:quickstart_training_projection]
dataset = Permutation.identity(table).select_columns(["power_score"])
dataloader = torch.utils.data.DataLoader(dataset, batch_size=1024)
# --8<-- [end:quickstart_training_projection]
batch = next(iter(dataloader))
assert set(batch) == {"power_score"}
assert batch["power_score"].tolist() == [3.5, 4.0, 3.0, 3.75]
@pytest.mark.asyncio
async def test_quickstart_async_api(db_path_factory):
db_uri = db_path_factory("quickstart_async_db")
import lancedb
async_db = await lancedb.connect_async(db_uri)
# --8<-- [start:quickstart_data_async]
data = [
{
"id": "1",
"name": "King Arthur",
"role": "King",
"description": "Leader of Camelot and wielder of Excalibur.",
"stats": {"strength": 4, "magic": 1, "leadership": 5, "wisdom": 4},
"vector": [0.7, 0.1, 0.9, 0.7],
},
{
"id": "2",
"name": "Merlin",
"role": "Wizard",
"description": "Advisor and prophet with deep magical knowledge.",
"stats": {"strength": 2, "magic": 5, "leadership": 4, "wisdom": 5},
"vector": [0.2, 0.9, 0.4, 0.9],
},
{
"id": "3",
"name": "Sir Lancelot",
"role": "Knight",
"description": "Legendary knight known for courage and combat skill.",
"stats": {"strength": 5, "magic": 1, "leadership": 3, "wisdom": 3},
"vector": [0.9, 0.1, 0.5, 0.4],
},
]
# --8<-- [end:quickstart_data_async]
# --8<-- [start:quickstart_create_table_async]
async_table = await async_db.create_table(
"characters",
data=data,
mode="overwrite",
)
# --8<-- [end:quickstart_create_table_async]
# --8<-- [start:quickstart_vector_search_1_async]
# Search for examples similar to a "wise magical advisor"
query_vector = [0.2, 0.8, 0.4, 0.9]
# Ensure you run `pip install polars` beforehand
async_result = await (
await async_table.search(query_vector)
).select(["name", "role", "description", "_distance"]).limit(2).to_polars()
print(async_result)
# --8<-- [end:quickstart_vector_search_1_async]
assert async_result.head(1)["name"][0] == "Merlin"
docs/web-tests/py/test_search.py
+730
View File
@@ -0,0 +1,730 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
# --8<-- [start:import-lancedb]
import lancedb
# --8<-- [end:import-lancedb]
# --8<-- [start:import-numpy]
from lancedb.query import BoostQuery, MatchQuery
import numpy as np
import pyarrow as pa
# --8<-- [end:import-numpy]
# --8<-- [start:import-datetime]
from datetime import datetime
# --8<-- [end:import-datetime]
# --8<-- [start:import-lancedb-pydantic]
from lancedb.pydantic import Vector, LanceModel
# --8<-- [end:import-lancedb-pydantic]
# --8<-- [start:import-pydantic-base-model]
from pydantic import BaseModel
# --8<-- [end:import-pydantic-base-model]
# --8<-- [start:import-lancedb-fts]
from lancedb.index import FTS
# --8<-- [end:import-lancedb-fts]
# --8<-- [start:import-os]
import os
# --8<-- [end:import-os]
# --8<-- [start:import-embeddings]
from lancedb.embeddings import get_registry
# --8<-- [end:import-embeddings]
import pytest
# --8<-- [start:class-definition]
class Metadata(BaseModel):
source: str
timestamp: datetime
class Document(BaseModel):
content: str
meta: Metadata
class LanceSchema(LanceModel):
id: str
vector: Vector(1536)
payload: Document
# --8<-- [end:class-definition]
def test_vector_search():
# --8<-- [start:exhaustive_search]
uri = "data/sample-lancedb"
db = lancedb.connect(uri)
data = [
{"vector": row, "item": f"item {i}"}
for i, row in enumerate(np.random.random((10_000, 1536)).astype("float32"))
]
tbl = db.create_table("vector_search", data=data, mode="overwrite")
tbl.search(np.random.random((1536))).limit(10).to_list()
# --8<-- [end:exhaustive_search]
# --8<-- [start:exhaustive_search_cosine]
tbl.search(np.random.random((1536))).distance_type("cosine").limit(10).to_list()
# --8<-- [end:exhaustive_search_cosine]
# --8<-- [start:create_table_with_nested_schema]
# Let's add 100 sample rows to our dataset
data = [
LanceSchema(
id=f"id{i}",
vector=np.random.randn(1536),
payload=Document(
content=f"document{i}",
meta=Metadata(source=f"source{i % 10}", timestamp=datetime.now()),
),
)
for i in range(100)
]
# Synchronous client
tbl = db.create_table("documents", data=data, mode="overwrite")
# --8<-- [end:create_table_with_nested_schema]
# --8<-- [start:search_result_as_pyarrow]
tbl.search(np.random.randn(1536)).to_arrow()
# --8<-- [end:search_result_as_pyarrow]
# --8<-- [start:search_result_as_pandas]
tbl.search(np.random.randn(1536)).to_pandas()
# --8<-- [end:search_result_as_pandas]
# --8<-- [start:search_result_as_pandas_flatten_true]
tbl.search(np.random.randn(1536)).to_pandas(flatten=True)
# --8<-- [end:search_result_as_pandas_flatten_true]
# --8<-- [start:search_result_as_pandas_flatten_1]
tbl.search(np.random.randn(1536)).to_pandas(flatten=1)
# --8<-- [end:search_result_as_pandas_flatten_1]
# --8<-- [start:search_result_as_list]
tbl.search(np.random.randn(1536)).to_list()
# --8<-- [end:search_result_as_list]
# --8<-- [start:search_result_as_pydantic]
tbl.search(np.random.randn(1536)).to_pydantic(LanceSchema)
# --8<-- [end:search_result_as_pydantic]
@pytest.mark.asyncio
async def test_vector_search_async():
# --8<-- [start:exhaustive_search_async]
uri = "data/sample-lancedb"
async_db = await lancedb.connect_async(uri)
data = [
{"vector": row, "item": f"item {i}"}
for i, row in enumerate(np.random.random((10_000, 1536)).astype("float32"))
]
async_tbl = await async_db.create_table(
"vector_search_async", data=data, mode="overwrite"
)
(await (await async_tbl.search(np.random.random((1536)))).limit(10).to_list())
# --8<-- [end:exhaustive_search_async]
# --8<-- [start:exhaustive_search_async_cosine]
(
await (await async_tbl.search(np.random.random((1536))))
.distance_type("cosine")
.limit(10)
.to_list()
)
# --8<-- [end:exhaustive_search_async_cosine]
# --8<-- [start:create_table_async_with_nested_schema]
# Let's add 100 sample rows to our dataset
data = [
LanceSchema(
id=f"id{i}",
vector=np.random.randn(1536),
payload=Document(
content=f"document{i}",
meta=Metadata(source=f"source{i % 10}", timestamp=datetime.now()),
),
)
for i in range(100)
]
async_tbl = await async_db.create_table(
"documents_async", data=data, mode="overwrite"
)
# --8<-- [end:create_table_async_with_nested_schema]
# --8<-- [start:search_result_async_as_pyarrow]
await (await async_tbl.search(np.random.randn(1536))).to_arrow()
# --8<-- [end:search_result_async_as_pyarrow]
# --8<-- [start:search_result_async_as_pandas]
await (await async_tbl.search(np.random.randn(1536))).to_pandas()
# --8<-- [end:search_result_async_as_pandas]
# --8<-- [start:search_result_async_as_list]
await (await async_tbl.search(np.random.randn(1536))).to_list()
# --8<-- [end:search_result_async_as_list]
def test_fts_fuzzy_query():
uri = "data/fuzzy-example"
db = lancedb.connect(uri)
table = db.create_table(
"my_table_fts_fuzzy",
data=pa.table(
{
"text": [
"fa",
"fo", # spellchecker:disable-line
"fob",
"focus",
"foo",
"food",
"foul",
]
}
),
mode="overwrite",
)
table.create_fts_index("text", replace=True)
results = table.search(MatchQuery("foo", "text", fuzziness=1)).to_pandas()
assert len(results) == 4
assert set(results["text"].to_list()) == {
"foo",
"fo", # 1 deletion # spellchecker:disable-line
"fob", # 1 substitution
"food", # 1 insertion
}
def test_fts_boost_query():
uri = "data/boost-example"
db = lancedb.connect(uri)
table = db.create_table(
"my_table_fts_boost",
data=pa.table(
{
"title": [
"The Hidden Gems of Travel",
"Exploring Nature's Wonders",
"Cultural Treasures Unveiled",
"The Nightlife Chronicles",
"Scenic Escapes and Challenges",
],
"desc": [
"A vibrant city with occasional traffic jams.",
"Beautiful landscapes but overpriced tourist spots.",
"Rich cultural heritage but humid summers.",
"Bustling nightlife but noisy streets.",
"Scenic views but limited public transport options.",
],
}
),
mode="overwrite",
)
table.create_fts_index("desc", replace=True)
results = table.search(
BoostQuery(
MatchQuery("beautiful, cultural, nightlife", "desc"),
MatchQuery("bad traffic jams, overpriced", "desc"),
),
).to_pandas()
# we will hit 3 results because the positive query has 3 hits
assert len(results) == 3
# the one containing "overpriced" will be negatively boosted,
# so it will be the last one
assert (
results["desc"].to_list()[2]
== "Beautiful landscapes but overpriced tourist spots."
)
def test_fts_native():
# --8<-- [start:basic_fts]
uri = "data/sample-lancedb"
db = lancedb.connect(uri)
table = db.create_table(
"my_table_fts",
data=[
{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"},
{"vector": [5.9, 26.5], "text": "There are several kittens playing"},
],
mode="overwrite",
)
table.create_fts_index("text")
table.search("puppy").limit(10).select(["text"]).to_list()
# [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}]
# ...
# --8<-- [end:basic_fts]
# --8<-- [start:fts_config_stem]
table.create_fts_index("text", tokenizer_name="en_stem", replace=True)
# --8<-- [end:fts_config_stem]
# --8<-- [start:fts_config_folding]
table.create_fts_index(
"text",
language="French",
stem=True,
ascii_folding=True,
replace=True,
)
# --8<-- [end:fts_config_folding]
# --8<-- [start:fts_prefiltering]
table.search("puppy").limit(10).where("text='foo'", prefilter=True).to_list()
# --8<-- [end:fts_prefiltering]
# --8<-- [start:fts_postfiltering]
table.search("puppy").limit(10).where("text='foo'", prefilter=False).to_list()
# --8<-- [end:fts_postfiltering]
# --8<-- [start:fts_with_position]
table.create_fts_index("text", with_position=True, replace=True)
# --8<-- [end:fts_with_position]
# --8<-- [start:fts_incremental_index]
table.add([{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}])
table.optimize()
# --8<-- [end:fts_incremental_index]
@pytest.mark.asyncio
async def test_fts_native_async():
# --8<-- [start:basic_fts_async]
uri = "data/sample-lancedb"
async_db = await lancedb.connect_async(uri)
async_tbl = await async_db.create_table(
"my_table_fts_async",
data=[
{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"},
{"vector": [5.9, 26.5], "text": "There are several kittens playing"},
],
mode="overwrite",
)
# async API uses our native FTS algorithm
await async_tbl.create_index("text", config=FTS())
await (await async_tbl.search("puppy")).select(["text"]).limit(10).to_list()
# [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}]
# ...
# --8<-- [end:basic_fts_async]
# --8<-- [start:fts_config_stem_async]
await async_tbl.create_index(
"text", config=FTS(language="English", stem=True, remove_stop_words=True)
)
# --8<-- [end:fts_config_stem_async]
# --8<-- [start:fts_config_folding_async]
await async_tbl.create_index(
"text", config=FTS(language="French", stem=True, ascii_folding=True)
)
# --8<-- [end:fts_config_folding_async]
# --8<-- [start:fts_prefiltering_async]
await (await async_tbl.search("puppy")).limit(10).where("text='foo'").to_list()
# --8<-- [end:fts_prefiltering_async]
# --8<-- [start:fts_postfiltering_async]
await (
(await async_tbl.search("puppy"))
.limit(10)
.where("text='foo'")
.postfilter()
.to_list()
)
# --8<-- [end:fts_postfiltering_async]
# --8<-- [start:fts_with_position_async]
await async_tbl.create_index("text", config=FTS(with_position=True))
# --8<-- [end:fts_with_position_async]
# --8<-- [start:fts_incremental_index_async]
await async_tbl.add([{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}])
await async_tbl.optimize()
# --8<-- [end:fts_incremental_index_async]
def _vectors(n, dim, seed=0, column="vector"):
rng = np.random.default_rng(seed)
return [
{column: rng.random(dim).astype("float32"), "id": i} for i in range(n)
]
def _build_indexed_table(db, name, dim=128, n=512):
tbl = db.create_table(name, _vectors(n, dim), mode="overwrite")
tbl.create_index(metric="cosine", num_partitions=4, num_sub_vectors=8)
return tbl
def test_vs_distance_metric_and_brute_force(tmp_db):
tbl = tmp_db.create_table("vs_metric", _vectors(64, 1536), mode="overwrite")
# --8<-- [start:configure_distance_metric]
tbl.search(np.random.random((1536))).distance_type("cosine").limit(10).to_list()
# --8<-- [end:configure_distance_metric]
# --8<-- [start:brute_force_search]
tbl.search(np.random.random((1536))).limit(3).to_list()
# --8<-- [end:brute_force_search]
def test_vs_select_vector_column(tmp_db):
db = tmp_db
# --8<-- [start:select_vector_column]
import pyarrow as pa
schema = pa.schema([
pa.field("id", pa.int32()),
pa.field(
"image",
pa.struct([pa.field("embedding", pa.list_(pa.float32(), 2))]),
),
])
table = db.create_table(
"nested",
data=[{"id": 0, "image": {"embedding": [0.0, 1.0]}}],
schema=schema,
)
# Inferred: the only vector leaf is `image.embedding`.
table.search([0.0, 1.0]).limit(1).to_list()
# Explicit: required when more than one vector column matches.
table.search([0.0, 1.0], vector_column_name="image.embedding").limit(1).to_list()
# --8<-- [end:select_vector_column]
def test_vs_index_nested_column(tmp_db):
dim = 16
schema = pa.schema(
[
pa.field("id", pa.int32()),
pa.field(
"image",
pa.struct([pa.field("embedding", pa.list_(pa.float32(), dim))]),
),
]
)
rng = np.random.default_rng(0)
data = [
{"id": i, "image": {"embedding": rng.random(dim).astype("float32").tolist()}}
for i in range(512)
]
table = tmp_db.create_table(
"nested_index", data=data, schema=schema, mode="overwrite"
)
# --8<-- [start:index_nested_column]
table.create_index(vector_column_name="image.embedding")
# --8<-- [end:index_nested_column]
assert table.list_indices()
def test_vs_indexed_queries(tmp_db):
table = _build_indexed_table(tmp_db, "vs_indexed", dim=128, n=512)
embedding = np.random.random(128)
# --8<-- [start:exact_vs_approximate_distances]
# Indexed ANN search without refinement (fast, approximate `_distance`)
fast_results = (
table.search(embedding)
.limit(10)
.to_pandas()
)
# Recompute distances on full vectors for reranked candidates
exact_distance_results = (
table.search(embedding)
.limit(10)
.refine_factor(1)
.to_pandas()
)
# Rerank a larger candidate set for better recall (higher latency)
higher_recall_results = (
table.search(embedding)
.limit(10)
.refine_factor(20)
.to_pandas()
)
# --8<-- [end:exact_vs_approximate_distances]
# --8<-- [start:bypass_vector_index]
table.search(embedding).bypass_vector_index().limit(5).to_pandas()
# --8<-- [end:bypass_vector_index]
assert len(fast_results) == 10
assert len(exact_distance_results) == 10
assert len(higher_recall_results) == 10
def test_vs_fast_search(tmp_db, monkeypatch):
table = _build_indexed_table(tmp_db, "vs_fast", dim=128, n=512)
embedding = np.random.random(128)
# `fast_search` is an Enterprise/async query flag; strip it so the snippet
# runs against a local OSS table without changing what readers see.
real_search = table.search
monkeypatch.setattr(
table,
"search",
lambda *a, **k: real_search(
*a, **{key: val for key, val in k.items() if key != "fast_search"}
),
)
# --8<-- [start:fast_search]
table.search(embedding, fast_search=True).limit(5).to_pandas()
# --8<-- [end:fast_search]
def test_vs_distance_range(tmp_db):
tbl = tmp_db.create_table("vs_distance_range", _vectors(256, 256), mode="overwrite")
# --8<-- [start:search_distance_range]
query = np.random.random(256)
# Search for the vectors within the range of [0.1, 0.5)
tbl.search(query).distance_range(0.1, 0.5).to_arrow()
# Search for the vectors with the distance less than 0.5
tbl.search(query).distance_range(upper_bound=0.5).to_arrow()
# Search for the vectors with the distance greater or equal to 0.1
tbl.search(query).distance_range(lower_bound=0.1).to_arrow()
# --8<-- [end:search_distance_range]
def test_vs_multivector_search(tmp_db):
schema = pa.schema(
[
pa.field("id", pa.int64()),
pa.field("vector", pa.list_(pa.list_(pa.float32(), 256))),
]
)
rng = np.random.default_rng(0)
data = [{"id": i, "vector": rng.random(size=(2, 256)).tolist()} for i in range(64)]
tbl = tmp_db.create_table(
"vs_multivector", data=data, schema=schema, mode="overwrite"
)
# --8<-- [start:multivector_search]
query_multi = np.random.random(size=(2, 256))
results_multi = tbl.search(query_multi).limit(5).to_pandas()
# --8<-- [end:multivector_search]
assert len(results_multi) <= 5
def test_vs_binary_search(tmp_db):
db = tmp_db
# --8<-- [start:search_binary_vectors]
import numpy as np
import pyarrow as pa
schema = pa.schema(
[
pa.field("id", pa.int64()),
# for dim=256, lance stores every 8 bits in a byte
# so the vector field should be a list of 256 / 8 = 32 bytes
pa.field("vector", pa.list_(pa.uint8(), 32)),
]
)
tbl = db.create_table("my_binary_vectors", schema=schema)
data = []
for i in range(1024):
vector = np.random.randint(0, 2, size=256)
# pack the binary vector into bytes to save space
packed_vector = np.packbits(vector)
data.append(
{
"id": i,
"vector": packed_vector,
}
)
tbl.add(data)
query = np.random.randint(0, 2, size=256)
packed_query = np.packbits(query)
tbl.search(packed_query).distance_type("hamming").to_arrow()
# --8<-- [end:search_binary_vectors]
def test_vs_enterprise_filtering(tmp_db, monkeypatch):
import sys
import types
dim = 384
rng = np.random.default_rng(0)
rows = [
{
"vector": rng.random(dim).astype("float32"),
"text": f"story {i}",
"keywords": f"kw{i}",
"label": i % 4,
}
for i in range(50)
]
tmp_db.create_table("lancedb-enterprise-quickstart", data=rows, mode="overwrite")
db = tmp_db
# Mock the Hugging Face dataset loader to avoid network downloads.
class _FakeDataset:
def __init__(self, n):
self._emb = [rng.random(dim).astype("float32") for _ in range(n)]
self._kw = [f"kw{i}" for i in range(n)]
def __getitem__(self, key):
if isinstance(key, str):
return {"keywords_embeddings": self._emb, "keywords": self._kw}[key]
return {"keywords": self._kw[key], "keywords_embeddings": self._emb[key]}
fake_datasets = types.ModuleType("datasets")
fake_datasets.load_dataset = lambda *a, **k: _FakeDataset(10)
monkeypatch.setitem(sys.modules, "datasets", fake_datasets)
# --8<-- [start:vector_search_prefilter]
from datasets import load_dataset
# Load query vector from dataset
query_dataset = load_dataset("sunhaozhepy/ag_news_sbert_keywords_embeddings", split="test[5000:5001]")
print(f"Query keywords: {query_dataset[0]['keywords']}")
query_embed = query_dataset["keywords_embeddings"][0]
# Open table and perform search
table_name = "lancedb-enterprise-quickstart"
table = db.open_table(table_name)
# Vector search with filters (pre-filtering is the default)
search_results = (
table.search(query_embed)
.where("label > 2")
.select(["text", "keywords", "label"])
.limit(5)
.to_pandas()
)
print("Search results (with pre-filtering):")
print(search_results)
# --8<-- [end:vector_search_prefilter]
# --8<-- [start:vector_search_postfilter]
results_post_filtered = (
table.search(query_embed)
.where("label > 1", prefilter=False)
.select(["text", "keywords", "label"])
.limit(5)
.to_pandas()
)
print("Vector search results with post-filter:")
print(results_post_filtered)
# --8<-- [end:vector_search_postfilter]
# --8<-- [start:batch_search]
# Load a batch of query embeddings
query_dataset = load_dataset(
"sunhaozhepy/ag_news_sbert_keywords_embeddings", split="test[5000:5005]"
)
query_embeds = query_dataset["keywords_embeddings"]
batch_results = table.search(query_embeds).limit(5).to_pandas()
print(batch_results)
# --8<-- [end:batch_search]
@pytest.mark.skip()
def test_hybrid_search():
# --8<-- [start:import-openai]
import openai
# --8<-- [end:import-openai]
# --8<-- [start:openai-embeddings]
# Ingest embedding function in LanceDB table
# Configuring the environment variable OPENAI_API_KEY
if "OPENAI_API_KEY" not in os.environ:
# OR set the key here as a variable
openai.api_key = "sk-..."
embeddings = get_registry().get("openai").create()
# --8<-- [end:openai-embeddings]
# --8<-- [start:class-Documents]
class Documents(LanceModel):
vector: Vector(embeddings.ndims()) = embeddings.VectorField()
text: str = embeddings.SourceField()
# --8<-- [end:class-Documents]
# --8<-- [start:basic_hybrid_search]
data = [
{"text": "rebel spaceships striking from a hidden base"},
{"text": "have won their first victory against the evil Galactic Empire"},
{"text": "during the battle rebel spies managed to steal secret plans"},
{"text": "to the Empire's ultimate weapon the Death Star"},
]
uri = "data/sample-lancedb"
db = lancedb.connect(uri)
table = db.create_table("documents", schema=Documents)
# ingest docs with auto-vectorization
table.add(data)
# Create a fts index before the hybrid search
table.create_fts_index("text")
# hybrid search with default re-ranker
table.search("flower moon", query_type="hybrid").to_pandas()
# --8<-- [end:basic_hybrid_search]
# --8<-- [start:hybrid_search_pass_vector_text]
vector_query = [0.1, 0.2, 0.3, 0.4, 0.5]
text_query = "flower moon"
(
table.search(query_type="hybrid")
.vector(vector_query)
.text(text_query)
.limit(5)
.to_pandas()
)
# --8<-- [end:hybrid_search_pass_vector_text]
@pytest.mark.skip
async def test_hybrid_search_async():
import openai
# --8<-- [start:openai-embeddings]
# Ingest embedding function in LanceDB table
# Configuring the environment variable OPENAI_API_KEY
if "OPENAI_API_KEY" not in os.environ:
# OR set the key here as a variable
openai.api_key = "sk-..."
embeddings = get_registry().get("openai").create()
# --8<-- [end:openai-embeddings]
# --8<-- [start:class-Documents]
class Documents(LanceModel):
vector: Vector(embeddings.ndims()) = embeddings.VectorField()
text: str = embeddings.SourceField()
# --8<-- [end:class-Documents]
# --8<-- [start:basic_hybrid_search_async]
uri = "data/sample-lancedb"
async_db = await lancedb.connect_async(uri)
data = [
{"text": "rebel spaceships striking from a hidden base"},
{"text": "have won their first victory against the evil Galactic Empire"},
{"text": "during the battle rebel spies managed to steal secret plans"},
{"text": "to the Empire's ultimate weapon the Death Star"},
]
async_tbl = await async_db.create_table("documents_async", schema=Documents)
# ingest docs with auto-vectorization
await async_tbl.add(data)
# Create a fts index before the hybrid search
await async_tbl.create_index("text", config=FTS())
text_query = "flower moon"
# hybrid search with default re-ranker
await (await async_tbl.search("flower moon", query_type="hybrid")).to_pandas()
# --8<-- [end:basic_hybrid_search_async]
# --8<-- [start:hybrid_search_pass_vector_text_async]
vector_query = [0.1, 0.2, 0.3, 0.4, 0.5]
text_query = "flower moon"
await (
async_tbl.query()
.nearest_to(vector_query)
.nearest_to_text(text_query)
.limit(5)
.to_pandas()
)
# --8<-- [end:hybrid_search_pass_vector_text_async]
docs/web-tests/py/test_storage.py
+162
View File
@@ -0,0 +1,162 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import lancedb
import pytest
class DummyTable:
def __init__(self, name: str, storage_options: dict | None = None):
self.name = name
self.storage_options = storage_options
class DummyConnection:
def __init__(self, uri: str, options: dict):
self.uri = uri
self.options = options
self.created_tables: list[DummyTable] = []
def create_table(
self,
name: str,
data,
storage_options: dict | None = None,
):
table = DummyTable(name, storage_options=storage_options)
self.created_tables.append(table)
return table
@pytest.fixture
def fake_connect(monkeypatch):
calls: list[DummyConnection] = []
def _fake_connect(uri: str, **kwargs):
conn = DummyConnection(uri, kwargs)
calls.append(conn)
return conn
monkeypatch.setattr(lancedb, "connect", _fake_connect)
return calls
def test_storage_snippets(fake_connect):
# --8<-- [start:storage_connect_s3]
db = lancedb.connect("s3://bucket/path")
# --8<-- [end:storage_connect_s3]
# --8<-- [start:storage_connect_gcs]
db = lancedb.connect("gs://bucket/path")
# --8<-- [end:storage_connect_gcs]
# --8<-- [start:storage_connect_azure]
db = lancedb.connect("az://bucket/path")
# --8<-- [end:storage_connect_azure]
# --8<-- [start:storage_connect_timeout]
db = lancedb.connect(
"s3://bucket/path",
storage_options={"timeout": "60s"},
)
# --8<-- [end:storage_connect_timeout]
# --8<-- [start:storage_table_timeout]
table = db.create_table(
"table",
[{"a": 1, "b": 2}],
storage_options={"timeout": "60s"},
)
# --8<-- [end:storage_table_timeout]
# --8<-- [start:storage_s3_ddb]
db = lancedb.connect(
"s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
)
# --8<-- [end:storage_s3_ddb]
# --8<-- [start:storage_s3_ddb_local]
db = lancedb.connect(
"s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
storage_options={
"endpoint": "http://localhost:4566",
"dynamodb_endpoint": "http://localhost:4566",
"allow_http": "true",
},
)
# --8<-- [end:storage_s3_ddb_local]
# --8<-- [start:storage_s3_sse_kms]
db = lancedb.connect(
"s3://bucket/path",
storage_options={
"aws_server_side_encryption": "aws:kms",
"aws_sse_kms_key_id": "<your-kms-key-id-or-arn>",
},
)
# --8<-- [end:storage_s3_sse_kms]
# --8<-- [start:storage_azure_sas]
db = lancedb.connect(
"az://my-container/my-database",
storage_options={
"azure_storage_account_name": "some-account",
"azure_storage_sas_token": "<sas-token>",
},
)
# --8<-- [end:storage_azure_sas]
# --8<-- [start:storage_s3_minio]
db = lancedb.connect(
"s3://bucket/path",
storage_options={
"region": "us-east-1",
"endpoint": "http://minio:9000",
},
)
# --8<-- [end:storage_s3_minio]
# --8<-- [start:storage_s3_express]
db = lancedb.connect(
"s3://my-bucket--use1-az4--x-s3/path",
storage_options={
"region": "us-east-1",
"s3_express": "true",
},
)
# --8<-- [end:storage_s3_express]
# --8<-- [start:storage_gcs_service_account]
db = lancedb.connect(
"gs://my-bucket/my-database",
storage_options={
"service_account": "path/to/service-account.json",
},
)
# --8<-- [end:storage_gcs_service_account]
# --8<-- [start:storage_azure_account]
db = lancedb.connect(
"az://my-container/my-database",
storage_options={
"account_name": "some-account",
"account_key": "some-key",
},
)
# --8<-- [end:storage_azure_account]
# --8<-- [start:storage_tigris_connect]
db = lancedb.connect(
"s3://your-bucket/path",
storage_options={
"endpoint": "https://t3.storage.dev",
"region": "auto",
},
)
# --8<-- [end:storage_tigris_connect]
assert len(fake_connect) == 13
assert all(
conn.uri.startswith(("s3://", "gs://", "az://", "s3+ddb://"))
for conn in fake_connect
)
docs/web-tests/py/test_tables.py
+1546
View File
File diff suppressed because it is too large Load Diff
docs/web-tests/rs/Cargo.lock
Generated
+7454
View File
File diff suppressed because it is too large Load Diff
docs/web-tests/rs/Cargo.toml
+43
View File
@@ -0,0 +1,43 @@
[package]
name = "rs"
version = "0.1.0"
edition = "2024"
[dependencies]
arrow-array = "58.3.0"
arrow-json = "58.3.0"
arrow-schema = "58.3.0"
futures = "0.3.31"
futures-util = "0.3.31"
lance-namespace = "7.0.0"
lancedb = { version = "0.31.0", features = ["polars"] }
polars = ">=0.37,<0.40.0"
polars-arrow = ">=0.37,<0.40.0"
serde = { version = "1", features = ["derive"] }
serde_json = "1.0.145"
tempfile = "3.23.0"
tokio = { version = "1", features = ["macros", "rt-multi-thread"] }
[[example]]
name = "basic_usage"
path = "basic_usage.rs"
[[example]]
name = "connection"
path = "connection.rs"
[[example]]
name = "quickstart"
path = "quickstart.rs"
[[example]]
name = "tables"
path = "tables.rs"
[[example]]
name = "multimodal"
path = "multimodal.rs"
[[example]]
name = "search"
path = "search.rs"
docs/web-tests/rs/basic_usage.rs
+346
View File
@@ -0,0 +1,346 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::fs;
use std::path::PathBuf;
use std::sync::Arc;
use serde::Deserialize;
type BatchIter = Box<dyn arrow_array::RecordBatchReader + Send>;
// --8<-- [start:basic_imports]
use arrow_array::types::Float32Type;
use arrow_array::{
FixedSizeListArray, Int8Array, Int16Array, RecordBatch, RecordBatchIterator, StringArray,
StructArray,
};
use arrow_schema::{DataType, Field, FieldRef, Schema};
use futures_util::TryStreamExt;
use lancedb::database::CreateTableMode;
use lancedb::query::{ExecutableQuery, QueryBase, Select};
use lancedb::{connect, table::NewColumnTransform};
// --8<-- [end:basic_imports]
#[derive(Debug, Clone, Deserialize)]
struct Stats {
strength: i8,
courage: i8,
magic: i8,
wisdom: i8,
}
#[derive(Debug, Clone, Deserialize)]
struct Character {
id: i16,
name: String,
role: String,
description: String,
vector: [f32; 4],
stats: Stats,
}
fn camelot_schema() -> Arc<Schema> {
Arc::new(Schema::new(vec![
Field::new("id", DataType::Int16, false),
Field::new("name", DataType::Utf8, false),
Field::new("role", DataType::Utf8, false),
Field::new("description", DataType::Utf8, false),
Field::new(
"vector",
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4),
false,
),
Field::new(
"stats",
DataType::Struct(arrow_schema::Fields::from(vec![
Arc::new(Field::new("strength", DataType::Int8, false)),
Arc::new(Field::new("courage", DataType::Int8, false)),
Arc::new(Field::new("magic", DataType::Int8, false)),
Arc::new(Field::new("wisdom", DataType::Int8, false)),
])),
false,
),
]))
}
fn characters_to_record_batch(schema: Arc<Schema>, characters: &[Character]) -> RecordBatch {
let ids = Int16Array::from_iter_values(characters.iter().map(|c| c.id));
let names = StringArray::from_iter_values(characters.iter().map(|c| c.name.as_str()));
let roles = StringArray::from_iter_values(characters.iter().map(|c| c.role.as_str()));
let descriptions =
StringArray::from_iter_values(characters.iter().map(|c| c.description.as_str()));
let vectors = FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
characters
.iter()
.map(|c| Some(c.vector.iter().copied().map(Some).collect::<Vec<_>>())),
4,
);
let strength = Int8Array::from_iter_values(characters.iter().map(|c| c.stats.strength));
let courage = Int8Array::from_iter_values(characters.iter().map(|c| c.stats.courage));
let magic = Int8Array::from_iter_values(characters.iter().map(|c| c.stats.magic));
let wisdom = Int8Array::from_iter_values(characters.iter().map(|c| c.stats.wisdom));
let stats_fields: Vec<FieldRef> = vec![
Arc::new(Field::new("strength", DataType::Int8, false)),
Arc::new(Field::new("courage", DataType::Int8, false)),
Arc::new(Field::new("magic", DataType::Int8, false)),
Arc::new(Field::new("wisdom", DataType::Int8, false)),
];
let stats = StructArray::new(
stats_fields.into(),
vec![
Arc::new(strength),
Arc::new(courage),
Arc::new(magic),
Arc::new(wisdom),
],
None,
);
RecordBatch::try_new(
schema,
vec![
Arc::new(ids),
Arc::new(names),
Arc::new(roles),
Arc::new(descriptions),
Arc::new(vectors),
Arc::new(stats),
],
)
.unwrap()
}
fn characters_to_reader(schema: Arc<Schema>, characters: &[Character]) -> BatchIter {
let batch = characters_to_record_batch(schema.clone(), characters);
Box::new(RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema))
}
fn camelot_json_path() -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("..")
.join("camelot.json")
}
#[tokio::main]
async fn main() {
let temp_dir = tempfile::tempdir().unwrap();
let uri = temp_dir.path().to_str().unwrap();
let db = connect(uri).execute().await.unwrap();
// --8<-- [start:data_load]
let data: Vec<Character> =
serde_json::from_str(&fs::read_to_string(camelot_json_path()).unwrap()).unwrap();
// --8<-- [end:data_load]
let schema = camelot_schema();
// --8<-- [start:basic_create_table]
let mut table = db
.create_table("camelot", characters_to_reader(schema.clone(), &data))
.mode(CreateTableMode::Overwrite)
.execute()
.await
.unwrap();
// --8<-- [end:basic_create_table]
assert_eq!(table.count_rows(None).await.unwrap(), 8);
// --8<-- [start:basic_open_table]
table = db.open_table("camelot").execute().await.unwrap();
// --8<-- [end:basic_open_table]
// --8<-- [start:basic_create_empty_table]
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int16, false),
Field::new("name", DataType::Utf8, false),
Field::new("role", DataType::Utf8, false),
Field::new("description", DataType::Utf8, false),
Field::new(
"vector",
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4),
false,
),
Field::new(
"stats",
DataType::Struct(arrow_schema::Fields::from(vec![
Arc::new(Field::new("strength", DataType::Int8, false)),
Arc::new(Field::new("courage", DataType::Int8, false)),
Arc::new(Field::new("magic", DataType::Int8, false)),
Arc::new(Field::new("wisdom", DataType::Int8, false)),
])),
false,
),
]));
db.create_empty_table("camelot_empty", schema)
.mode(CreateTableMode::Overwrite)
.execute()
.await
.unwrap();
// --8<-- [end:basic_create_empty_table]
db.drop_table("camelot_empty", &[]).await.unwrap();
// --8<-- [start:basic_add_data]
let magical_characters = vec![
Character {
id: 9,
name: "Morgan le Fay".to_string(),
role: "Sorceress".to_string(),
description: "A powerful enchantress, Arthur's half-sister, and a complex figure who oscillates between aiding and opposing Camelot.".to_string(),
vector: [0.10, 0.84, 0.25, 0.70],
stats: Stats {
strength: 2,
courage: 3,
magic: 5,
wisdom: 4,
},
},
Character {
id: 10,
name: "The Lady of the Lake".to_string(),
role: "Mystical Guardian".to_string(),
description: "A mysterious supernatural figure associated with Avalon, known for giving Arthur the sword Excalibur.".to_string(),
vector: [0.00, 0.90, 0.58, 0.88],
stats: Stats {
strength: 2,
courage: 3,
magic: 5,
wisdom: 5,
},
},
];
table
.add(characters_to_reader(camelot_schema(), &magical_characters))
.execute()
.await
.unwrap();
// --8<-- [end:basic_add_data]
// --8<-- [start:basic_vector_search]
let query_vector = [0.03, 0.85, 0.61, 0.90];
let result = table
.query()
.nearest_to(&query_vector)
.unwrap()
.limit(5)
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
println!("{result:?}");
// --8<-- [end:basic_vector_search]
// --8<-- [start:basic_add_columns]
table
.add_columns(
NewColumnTransform::SqlExpressions(vec![(
"power".to_string(),
"cast(((stats.strength + stats.courage + stats.magic + stats.wisdom) / 4.0) as float)"
.to_string(),
)]),
None,
)
.await
.unwrap();
// --8<-- [end:basic_add_columns]
// --8<-- [start:basic_vector_search_q1]
// Who are the characters similar to "wizard"?
let query_vector_1 = [0.03, 0.85, 0.61, 0.90];
let r1 = table
.query()
.nearest_to(&query_vector_1)
.unwrap()
.limit(5)
.select(Select::Columns(vec![
"name".to_string(),
"role".to_string(),
"description".to_string(),
]))
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
println!("{r1:?}");
// --8<-- [end:basic_vector_search_q1]
// --8<-- [start:basic_vector_search_q2]
// Who are the characters similar to "wizard" with high magic stats?
let query_vector_2 = [0.03, 0.85, 0.61, 0.90];
let r2 = table
.query()
.nearest_to(&query_vector_2)
.unwrap()
.only_if("stats.magic > 3")
.select(Select::Columns(vec![
"name".to_string(),
"role".to_string(),
"description".to_string(),
]))
.limit(5)
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
println!("{r2:?}");
// --8<-- [end:basic_vector_search_q2]
// --8<-- [start:basic_vector_search_q3]
// Who are the strongest characters?
let r3 = table
.query()
.only_if("stats.strength > 3")
.select(Select::Columns(vec![
"name".to_string(),
"role".to_string(),
"description".to_string(),
]))
.limit(5)
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
println!("{r3:?}");
// --8<-- [end:basic_vector_search_q3]
// --8<-- [start:basic_vector_search_q4]
// Who are the strongest characters?
let r4 = table
.query()
.select(Select::Columns(vec![
"name".to_string(),
"role".to_string(),
"description".to_string(),
"power".to_string(),
]))
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
println!("{r4:?}");
// --8<-- [end:basic_vector_search_q4]
// --8<-- [start:basic_drop_columns]
table.drop_columns(&["power"]).await.unwrap();
// --8<-- [end:basic_drop_columns]
// --8<-- [start:basic_delete_rows]
table.delete("role = 'Traitor Knight'").await.unwrap();
// --8<-- [end:basic_delete_rows]
// --8<-- [start:basic_drop_table]
db.drop_table("camelot", &[]).await.unwrap();
// --8<-- [end:basic_drop_table]
}
docs/web-tests/rs/bedrock.rs
+92
View File
@@ -0,0 +1,92 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::{iter::once, sync::Arc};
use arrow_array::{Float64Array, Int32Array, RecordBatch, RecordBatchIterator, StringArray};
use arrow_schema::{DataType, Field, Schema};
use aws_config::Region;
use aws_sdk_bedrockruntime::Client;
use futures::StreamExt;
use lancedb::{
arrow::IntoArrow,
connect,
embeddings::{bedrock::BedrockEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},
query::{ExecutableQuery, QueryBase},
Result,
};
#[tokio::main]
async fn main() -> Result<()> {
let tempdir = tempfile::tempdir().unwrap();
let tempdir = tempdir.path().to_str().unwrap();
// create Bedrock embedding function
let region: String = "us-east-1".to_string();
let config = aws_config::defaults(aws_config::BehaviorVersion::latest())
.region(Region::new(region))
.load()
.await;
let embedding = Arc::new(BedrockEmbeddingFunction::new(
Client::new(&config), // AWS Region
));
let db = connect(tempdir).execute().await?;
db.embedding_registry()
.register("bedrock", embedding.clone())?;
let table = db
.create_table("vectors", make_data())
.add_embedding(EmbeddingDefinition::new(
"text",
"bedrock",
Some("embeddings"),
))?
.execute()
.await?;
// execute vector search
let query = Arc::new(StringArray::from_iter_values(once("something warm")));
let query_vector = embedding.compute_query_embeddings(query)?;
let mut results = table
.vector_search(query_vector)?
.limit(1)
.execute()
.await?;
let rb = results.next().await.unwrap()?;
let out = rb
.column_by_name("text")
.unwrap()
.as_any()
.downcast_ref::<StringArray>()
.unwrap();
let text = out.iter().next().unwrap().unwrap();
println!("Closest match: {}", text);
Ok(())
}
fn make_data() -> impl IntoArrow {
let schema = Schema::new(vec![
Field::new("id", DataType::Int32, true),
Field::new("text", DataType::Utf8, false),
Field::new("price", DataType::Float64, false),
]);
let id = Int32Array::from(vec![1, 2, 3, 4]);
let text = StringArray::from_iter_values(vec![
"Black T-Shirt",
"Leather Jacket",
"Winter Parka",
"Hooded Sweatshirt",
]);
let price = Float64Array::from(vec![10.0, 50.0, 100.0, 30.0]);
let schema = Arc::new(schema);
let rb = RecordBatch::try_new(
schema.clone(),
vec![Arc::new(id), Arc::new(text), Arc::new(price)],
)
.unwrap();
Box::new(RecordBatchIterator::new(vec![Ok(rb)], schema))
}
docs/web-tests/rs/connection.rs
+134
View File
@@ -0,0 +1,134 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::path::PathBuf;
use lancedb::connect;
// --8<-- [start:connect]
async fn connect_example(uri: &str) {
let db = connect(uri).execute().await.unwrap();
let _ = db;
}
// --8<-- [end:connect]
#[tokio::main]
async fn main() {
let temp_dir = tempfile::tempdir().unwrap();
let uri = temp_dir.path().join("ex_lancedb");
connect_example(uri.to_str().unwrap()).await;
// Keep the enterprise snippet in this file, but don't run it in CI.
let _ = connect_enterprise_quickstart_config();
let _ = connect_object_storage_config();
}
fn connect_enterprise_quickstart_config() -> (String, String, String, String) {
// --8<-- [start:connect_enterprise_quickstart]
let uri = "db://your-database-uri";
let api_key = "your-api-key";
let region = "us-east-1";
let host_override = "https://your-enterprise-endpoint.com";
// --8<-- [end:connect_enterprise_quickstart]
(
uri.to_string(),
api_key.to_string(),
region.to_string(),
host_override.to_string(),
)
}
fn connect_object_storage_config() -> &'static str {
// --8<-- [start:connect_object_storage]
let uri = "s3://your-bucket/path";
// You can also use "gs://your-bucket/path" or "az://your-container/path".
// --8<-- [end:connect_object_storage]
uri
}
async fn namespace_table_ops_example(uri: &str) -> lancedb::Result<()> {
// --8<-- [start:namespace_table_ops]
let conn = connect(uri).execute().await?;
let search_namespace = vec!["prod".to_string(), "search".to_string()];
let recommendations_namespace = vec!["prod".to_string(), "recommendations".to_string()];
let schema = std::sync::Arc::new(arrow_schema::Schema::new(vec![
arrow_schema::Field::new("id", arrow_schema::DataType::Int64, false),
]));
conn.create_empty_table("user", schema.clone())
.namespace(search_namespace.clone())
.execute()
.await?;
conn.create_empty_table("user", schema)
.namespace(recommendations_namespace.clone())
.execute()
.await?;
let search_table_names = conn
.table_names()
.namespace(search_namespace)
.execute()
.await?;
let recommendation_table_names = conn
.table_names()
.namespace(recommendations_namespace)
.execute()
.await?;
println!("{search_table_names:?}"); // ["user"]
println!("{recommendation_table_names:?}"); // ["user"]
// --8<-- [end:namespace_table_ops]
Ok(())
}
async fn namespace_admin_ops_example() -> lancedb::Result<()> {
// --8<-- [start:namespace_admin_ops]
let mut properties = std::collections::HashMap::new();
properties.insert("root".to_string(), "./local_lancedb".to_string());
let db = lancedb::connect_namespace("dir", properties).execute().await?;
let namespace = vec!["prod".to_string(), "search".to_string()];
db.create_namespace(lance_namespace::models::CreateNamespaceRequest {
id: Some(vec!["prod".to_string()]),
..Default::default()
})
.await?;
db.create_namespace(lance_namespace::models::CreateNamespaceRequest {
id: Some(namespace.clone()),
..Default::default()
})
.await?;
let child_namespaces = db
.list_namespaces(lance_namespace::models::ListNamespacesRequest {
id: Some(vec!["prod".to_string()]),
..Default::default()
})
.await?;
println!(
"Child namespaces under {:?}: {:?}",
namespace, child_namespaces
);
// Child namespaces under ["prod", "search"]: ["search"]
db.drop_namespace(lance_namespace::models::DropNamespaceRequest {
id: Some(namespace.clone()),
..Default::default()
})
.await?;
db.drop_namespace(lance_namespace::models::DropNamespaceRequest {
id: Some(vec!["prod".to_string()]),
..Default::default()
})
.await?;
// --8<-- [end:namespace_admin_ops]
Ok(())
}
#[allow(dead_code)]
fn repo_root() -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..")
}
docs/web-tests/rs/embedding.rs
+182
View File
@@ -0,0 +1,182 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// --8<-- [start:openai_embeddings]
use std::{iter::once, sync::Arc};
use arrow_array::{record_batch, StringArray};
use arrow_schema::{DataType, Field, Schema};
use futures::StreamExt;
use lancedb::{
connect,
embeddings::{openai::OpenAIEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},
query::{ExecutableQuery, QueryBase},
Result,
};
#[tokio::main]
async fn main() -> Result<()> {
let db = connect("./mydb").execute().await?;
let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY is not set");
let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model(
api_key,
"text-embedding-3-large",
)?);
db.embedding_registry().register("openai", embedding.clone())?;
let schema = Arc::new(Schema::new(vec![Field::new("text", DataType::Utf8, false)]));
let table = db
.create_empty_table("mytable", schema)
.add_embedding(EmbeddingDefinition::new("text", "openai", Some("vector")))?
.execute()
.await?;
table
.add(record_batch!(("text", Utf8, ["This is a test.", "Another example."]))?)
.execute()
.await?;
let query = Arc::new(StringArray::from_iter_values(once("test example")));
let query_vector = embedding.compute_query_embeddings(query)?;
let mut results = table.vector_search(query_vector)?.limit(5).execute().await?;
while let Some(batch) = results.next().await {
println!("{:?}", batch?);
}
Ok(())
}
// --8<-- [end:openai_embeddings]
// --8<-- [start:create_embedding_function]
use std::sync::Arc;
use lancedb::embeddings::openai::OpenAIEmbeddingFunction;
let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY is not set");
let embedding = Arc::new(
OpenAIEmbeddingFunction::new_with_model(api_key, "text-embedding-3-small")
.expect("failed to create OpenAI embedding function"),
);
// --8<-- [end:create_embedding_function]
// --8<-- [start:manual_query_embeddings]
use std::{iter::once, sync::Arc};
use arrow_array::{record_batch, StringArray};
use arrow_schema::{DataType, Field, Schema};
use futures::StreamExt;
use lancedb::{
connect,
embeddings::{openai::OpenAIEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},
query::{ExecutableQuery, QueryBase},
Result,
};
#[tokio::main]
async fn main() -> Result<()> {
let db = connect("./mydb").execute().await?;
let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY is not set");
let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model(
api_key,
"text-embedding-3-large",
)?);
db.embedding_registry().register("openai", embedding.clone())?;
let schema = Arc::new(Schema::new(vec![Field::new("text", DataType::Utf8, false)]));
let table = db
.create_empty_table("mytable", schema)
.add_embedding(EmbeddingDefinition::new("text", "openai", Some("vector")))?
.execute()
.await?;
table
.add(record_batch!(("text", Utf8, ["This is a test.", "Another example."]))?)
.execute()
.await?;
// Manually generate embeddings for the query (Enterprise path)
let query = Arc::new(StringArray::from_iter_values(once("test example")));
let query_vector = embedding.compute_query_embeddings(query)?;
// --8<-- [start:manual_query_search]
// query_vector is assumed to already be generated by your embedding function
let mut results = table.vector_search(query_vector)?.limit(5).execute().await?;
while let Some(batch) = results.next().await {
println!("{:?}", batch?);
}
// --8<-- [end:manual_query_search]
Ok(())
}
// --8<-- [end:manual_query_embeddings]
// --8<-- [start:embedding_function]
use std::{borrow::Cow, sync::Arc};
use arrow_array::{Array, FixedSizeListArray, Float32Array};
use arrow_schema::{DataType, Field, Schema};
use lancedb::{
connect,
embeddings::{EmbeddingDefinition, EmbeddingFunction},
Result,
};
#[derive(Debug, Clone)]
struct MyTextEmbedder {
dim: usize,
}
impl EmbeddingFunction for MyTextEmbedder {
fn name(&self) -> &str {
"my-embedder"
}
fn source_type(&self) -> Result<Cow<'_, DataType>> {
Ok(Cow::Owned(DataType::Utf8))
}
fn dest_type(&self) -> Result<Cow<'_, DataType>> {
Ok(Cow::Owned(DataType::new_fixed_size_list(
DataType::Float32,
self.dim as i32,
true,
)))
}
fn compute_source_embeddings(&self, source: Arc<dyn Array>) -> Result<Arc<dyn Array>> {
let values = Arc::new(Float32Array::from(vec![1.0f32; source.len() * self.dim]));
let field = Arc::new(Field::new("item", DataType::Float32, true));
Ok(Arc::new(FixedSizeListArray::new(
field,
self.dim as i32,
values,
None,
)))
}
fn compute_query_embeddings(&self, _input: Arc<dyn Array>) -> Result<Arc<dyn Array>> {
unimplemented!()
}
}
#[tokio::main]
async fn main() -> Result<()> {
let db = connect("./mydb").execute().await?;
db.embedding_registry()
.register("my-embedder", Arc::new(MyTextEmbedder { dim: 3 }))?;
let schema = Arc::new(Schema::new(vec![Field::new("text", DataType::Utf8, false)]));
db.create_empty_table("mytable", schema)
.add_embedding(EmbeddingDefinition::new(
"text",
"my-embedder",
Some("vector"),
))?
.execute()
.await?;
Ok(())
}
// --8<-- [end:embedding_function]
docs/web-tests/rs/full_text_search.rs
+103
View File
@@ -0,0 +1,103 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::sync::Arc;
use arrow_array::{Int32Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray};
use arrow_schema::{DataType, Field, Schema};
use futures::TryStreamExt;
use lance_index::scalar::FullTextSearchQuery;
use lancedb::connection::Connection;
use lancedb::index::scalar::FtsIndexBuilder;
use lancedb::index::Index;
use lancedb::query::{ExecutableQuery, QueryBase};
use lancedb::{connect, Result, Table};
use rand::random;
#[tokio::main]
async fn main() -> Result<()> {
if std::path::Path::new("data").exists() {
std::fs::remove_dir_all("data").unwrap();
}
let uri = "data/sample-lancedb";
let db = connect(uri).execute().await?;
let tbl = create_table(&db).await?;
create_index(&tbl).await?;
search_index(&tbl).await?;
Ok(())
}
fn create_some_records() -> Result<Box<dyn RecordBatchReader + Send>> {
const TOTAL: usize = 1000;
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int32, false),
Field::new("doc", DataType::Utf8, true),
]));
let words = random_word::all(random_word::Lang::En)
.iter()
.step_by(1024)
.take(500)
.copied()
.collect::<Vec<_>>();
let n_terms = 3;
let batches = RecordBatchIterator::new(
vec![RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Int32Array::from_iter_values(0..TOTAL as i32)),
Arc::new(StringArray::from_iter_values((0..TOTAL).map(|_| {
(0..n_terms)
.map(|_| words[random::<usize>() % words.len()])
.collect::<Vec<_>>()
.join(" ")
}))),
],
)
.unwrap()]
.into_iter()
.map(Ok),
schema.clone(),
);
Ok(Box::new(batches))
}
async fn create_table(db: &Connection) -> Result<Table> {
let initial_data: Box<dyn RecordBatchReader + Send> = create_some_records()?;
let tbl = db.create_table("my_table", initial_data).execute().await?;
Ok(tbl)
}
async fn create_index(table: &Table) -> Result<()> {
table
.create_index(&["doc"], Index::FTS(FtsIndexBuilder::default()))
.execute()
.await?;
Ok(())
}
async fn search_index(table: &Table) -> Result<()> {
let words = random_word::all(random_word::Lang::En)
.iter()
.step_by(1024)
.take(500)
.copied()
.collect::<Vec<_>>();
let query = words[0].to_owned();
println!("Searching for: {}", query);
let mut results = table
.query()
.full_text_search(FullTextSearchQuery::new(words[0].to_owned()))
.select(lancedb::query::Select::Columns(vec!["doc".to_owned()]))
.limit(10)
.execute()
.await?;
while let Some(batch) = results.try_next().await? {
println!("{:?}", batch);
}
Ok(())
}
docs/web-tests/rs/ivf_pq.rs
+154
View File
@@ -0,0 +1,154 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
//! This example demonstrates setting advanced parameters when building an IVF PQ index
//!
//! Snippets from this example are used in the documentation on ANN indices.
use std::sync::Arc;
use arrow_array::types::Float32Type;
use arrow_array::{
FixedSizeListArray, Int32Array, RecordBatch, RecordBatchIterator, RecordBatchReader,
};
use arrow_schema::{DataType, Field, Schema};
use futures::TryStreamExt;
use lancedb::connection::Connection;
use lancedb::index::vector::IvfPqIndexBuilder;
use lancedb::index::Index;
use lancedb::query::{ExecutableQuery, QueryBase};
use lancedb::{connect, DistanceType, Result, Table};
#[tokio::main]
async fn main() -> Result<()> {
if std::path::Path::new("data").exists() {
std::fs::remove_dir_all("data").unwrap();
}
let uri = "data/sample-lancedb";
let db = connect(uri).execute().await?;
let tbl = create_table(&db).await?;
create_index(&tbl).await?;
search_index(&tbl).await?;
Ok(())
}
fn create_some_records() -> Result<Box<dyn RecordBatchReader + Send>> {
const TOTAL: usize = 1000;
const DIM: usize = 128;
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int32, false),
Field::new(
"vector",
DataType::FixedSizeList(
Arc::new(Field::new("item", DataType::Float32, true)),
DIM as i32,
),
true,
),
]));
// Create a RecordBatch stream.
let batches = RecordBatchIterator::new(
vec![RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Int32Array::from_iter_values(0..TOTAL as i32)),
Arc::new(
FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
(0..TOTAL).map(|_| Some(vec![Some(1.0); DIM])),
DIM as i32,
),
),
],
)
.unwrap()]
.into_iter()
.map(Ok),
schema.clone(),
);
Ok(Box::new(batches))
}
async fn create_table(db: &Connection) -> Result<Table> {
let initial_data: Box<dyn RecordBatchReader + Send> = create_some_records()?;
let tbl = db
.create_table("my_table", initial_data)
.execute()
.await
.unwrap();
Ok(tbl)
}
async fn create_index(table: &Table) -> Result<()> {
// --8<-- [start:create_index]
// For this example, `table` is a lancedb::Table with a column named
// "vector" that is a vector column with dimension 128.
// By default, if the column "vector" appears to be a vector column,
// then an IVF_PQ index with reasonable defaults is created.
table
.create_index(&["vector"], Index::Auto)
.execute()
.await?;
// For advanced cases, it is also possible to specifically request an
// IVF_PQ index and provide custom parameters.
table
.create_index(
&["vector"],
Index::IvfPq(
// Here we specify advanced indexing parameters. In this case
// we are creating an index that my have better recall than the
// default but is also larger and slower.
IvfPqIndexBuilder::default()
// This overrides the default distance type of l2
.distance_type(DistanceType::Cosine)
// With 1000 rows this have been ~31 by default
.num_partitions(50)
// With dimension 128 this would have been 8 by default
.num_sub_vectors(16),
),
)
.execute()
.await?;
// --8<-- [end:create_index]
Ok(())
}
async fn search_index(table: &Table) -> Result<()> {
// --8<-- [start:search1]
let query_vector = [1.0; 128];
// By default the index will find the 10 closest results using default
// search parameters that give a reasonable tradeoff between accuracy
// and search latency
let mut results = table
.vector_search(&query_vector)?
// Note: you should always set the distance_type to match the value used
// to train the index
.distance_type(DistanceType::Cosine)
.execute()
.await?;
while let Some(batch) = results.try_next().await? {
println!("{:?}", batch);
}
// We can also provide custom search parameters. Here we perform a
// slower but more accurate search
let mut results = table
.vector_search(&query_vector)?
.distance_type(DistanceType::Cosine)
// Override the default of 10 to get more rows
.limit(15)
// Override the default of 20 to search more partitions
.nprobes(30)
// Override the default of None to apply a refine step
.refine_factor(1)
.execute()
.await?;
while let Some(batch) = results.try_next().await? {
println!("{:?}", batch);
}
Ok(())
// --8<-- [end:search1]
}
docs/web-tests/rs/multimodal.rs
+178
View File
@@ -0,0 +1,178 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// --8<-- [start:multimodal_imports]
use std::collections::HashMap;
use std::sync::Arc;
use arrow_array::types::Float32Type;
use arrow_array::{
BinaryArray, FixedSizeListArray, Int32Array, Int64Array, LargeBinaryArray, RecordBatch,
RecordBatchIterator, StringArray,
};
use arrow_schema::{DataType, Field, Schema};
use futures_util::TryStreamExt;
use lancedb::connect;
use lancedb::database::CreateTableMode;
use lancedb::query::{ExecutableQuery, QueryBase};
// --8<-- [end:multimodal_imports]
#[tokio::main]
async fn main() {
let temp_dir = tempfile::tempdir().unwrap();
let db_uri = temp_dir.path().to_str().unwrap().to_string();
let db = connect(&db_uri).execute().await.unwrap();
// --8<-- [start:create_dummy_data]
let create_dummy_image = |color: u8| -> Vec<u8> {
let mut png_like = vec![137, 80, 78, 71, 13, 10, 26, 10];
png_like.push(color);
png_like
};
let data = vec![
(
1_i32,
"red_square.png",
vec![0.1_f32; 128],
create_dummy_image(1),
"red",
),
(
2_i32,
"blue_square.png",
vec![0.2_f32; 128],
create_dummy_image(2),
"blue",
),
];
// --8<-- [end:create_dummy_data]
// --8<-- [start:define_schema]
let schema = Schema::new(vec![
Field::new("id", DataType::Int32, false),
Field::new("filename", DataType::Utf8, false),
Field::new(
"vector",
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 128),
false,
),
Field::new("image_blob", DataType::Binary, false),
Field::new("label", DataType::Utf8, false),
]);
// --8<-- [end:define_schema]
// --8<-- [start:ingest_data]
let schema = Arc::new(schema);
let image_batch = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Int32Array::from_iter_values(data.iter().map(|row| row.0))),
Arc::new(StringArray::from_iter_values(data.iter().map(|row| row.1))),
Arc::new(
FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
data.iter()
.map(|row| Some(row.2.iter().copied().map(Some).collect::<Vec<_>>())),
128,
),
),
Arc::new(BinaryArray::from_iter_values(
data.iter().map(|row| row.3.as_slice()),
)),
Arc::new(StringArray::from_iter_values(data.iter().map(|row| row.4))),
],
)
.unwrap();
let image_reader: Box<dyn arrow_array::RecordBatchReader + Send> =
Box::new(RecordBatchIterator::new(vec![Ok(image_batch)].into_iter(), schema.clone()));
let table = db
.create_table("images", image_reader)
.mode(CreateTableMode::Overwrite)
.execute()
.await
.unwrap();
// --8<-- [end:ingest_data]
assert_eq!(table.count_rows(None).await.unwrap(), 2);
// --8<-- [start:search_data]
let query_vector = vec![0.1_f32; 128];
let results = table
.query()
.nearest_to(query_vector)
.unwrap()
.limit(1)
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
// --8<-- [end:search_data]
// --8<-- [start:process_results]
for batch in &results {
let filenames = batch
.column_by_name("filename")
.unwrap()
.as_any()
.downcast_ref::<StringArray>()
.unwrap();
let images = batch
.column_by_name("image_blob")
.unwrap()
.as_any()
.downcast_ref::<BinaryArray>()
.unwrap();
for row in 0..batch.num_rows() {
let image_bytes = images.value(row);
println!(
"Retrieved image: {}, Byte length: {}",
filenames.value(row),
image_bytes.len()
);
}
}
// --8<-- [end:process_results]
let search_rows: usize = results.iter().map(|batch| batch.num_rows()).sum();
assert_eq!(search_rows, 1);
// --8<-- [start:blob_api_schema]
let blob_metadata = HashMap::from([(
"lance-encoding:blob".to_string(),
"true".to_string(),
)]);
let blob_schema = Schema::new(vec![
Field::new("id", DataType::Int64, false),
Field::new("video", DataType::LargeBinary, true).with_metadata(blob_metadata),
]);
// --8<-- [end:blob_api_schema]
// --8<-- [start:blob_api_ingest]
let blob_rows = vec![
(1_i64, b"fake_video_bytes_1".to_vec()),
(2_i64, b"fake_video_bytes_2".to_vec()),
];
let blob_schema = Arc::new(blob_schema);
let blob_batch = RecordBatch::try_new(
blob_schema.clone(),
vec![
Arc::new(Int64Array::from_iter_values(blob_rows.iter().map(|row| row.0))),
Arc::new(LargeBinaryArray::from_iter_values(
blob_rows.iter().map(|row| row.1.as_slice()),
)),
],
)
.unwrap();
let blob_reader: Box<dyn arrow_array::RecordBatchReader + Send> =
Box::new(RecordBatchIterator::new(vec![Ok(blob_batch)].into_iter(), blob_schema));
let blob_table = db
.create_table("videos", blob_reader)
.mode(CreateTableMode::Overwrite)
.execute()
.await
.unwrap();
// --8<-- [end:blob_api_ingest]
assert_eq!(blob_table.count_rows(None).await.unwrap(), 2);
}
docs/web-tests/rs/openai.rs
+89
View File
@@ -0,0 +1,89 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// --8<-- [start:imports]
use std::{iter::once, sync::Arc};
use arrow_array::{Float64Array, Int32Array, RecordBatch, RecordBatchIterator, StringArray};
use arrow_schema::{DataType, Field, Schema};
use futures::StreamExt;
use lancedb::{
arrow::IntoArrow,
connect,
embeddings::{openai::OpenAIEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},
query::{ExecutableQuery, QueryBase},
Result,
};
// --8<-- [end:imports]
// --8<-- [start:openai_embeddings]
#[tokio::main]
async fn main() -> Result<()> {
let tempdir = tempfile::tempdir().unwrap();
let tempdir = tempdir.path().to_str().unwrap();
let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY is not set");
let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model(
api_key,
"text-embedding-3-large",
)?);
let db = connect(tempdir).execute().await?;
db.embedding_registry()
.register("openai", embedding.clone())?;
let table = db
.create_table("vectors", make_data())
.add_embedding(EmbeddingDefinition::new(
"text",
"openai",
Some("embeddings"),
))?
.execute()
.await?;
let query = Arc::new(StringArray::from_iter_values(once("something warm")));
let query_vector = embedding.compute_query_embeddings(query)?;
let mut results = table
.vector_search(query_vector)?
.limit(1)
.execute()
.await?;
let rb = results.next().await.unwrap()?;
let out = rb
.column_by_name("text")
.unwrap()
.as_any()
.downcast_ref::<StringArray>()
.unwrap();
let text = out.iter().next().unwrap().unwrap();
println!("Closest match: {}", text);
Ok(())
}
// --8<-- [end:openai_embeddings]
fn make_data() -> impl IntoArrow {
let schema = Schema::new(vec![
Field::new("id", DataType::Int32, true),
Field::new("text", DataType::Utf8, false),
Field::new("price", DataType::Float64, false),
]);
let id = Int32Array::from(vec![1, 2, 3, 4]);
let text = StringArray::from_iter_values(vec![
"Black T-Shirt",
"Leather Jacket",
"Winter Parka",
"Hooded Sweatshirt",
]);
let price = Float64Array::from(vec![10.0, 50.0, 100.0, 30.0]);
let schema = Arc::new(schema);
let rb = RecordBatch::try_new(
schema.clone(),
vec![Arc::new(id), Arc::new(text), Arc::new(price)],
)
.unwrap();
Box::new(RecordBatchIterator::new(vec![Ok(rb)], schema))
}
docs/web-tests/rs/quickstart.rs
+347
View File
@@ -0,0 +1,347 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::sync::Arc;
use arrow_array::types::Float32Type;
use arrow_array::{
FixedSizeListArray, Int8Array, LargeStringArray, RecordBatch, RecordBatchIterator, StructArray,
};
use arrow_schema::{DataType, Field, FieldRef, Schema};
use lancedb::arrow::IntoPolars;
use lancedb::database::CreateTableMode;
use lancedb::query::{ExecutableQuery, QueryBase, Select};
use lancedb::{connect, table::NewColumnTransform};
use polars::prelude::DataFrame;
use serde::{Deserialize, Serialize};
// --8<-- [start:quickstart_define_struct]
// Define structs representing the data schema
#[derive(Debug, Clone, Serialize, Deserialize)]
struct Stats {
strength: i8,
magic: i8,
leadership: i8,
wisdom: i8,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
struct Character {
id: String,
name: String,
role: String,
description: String,
stats: Stats,
vector: [f32; 4],
}
fn characters_schema() -> Arc<Schema> {
Arc::new(Schema::new(vec![
Field::new("id", DataType::LargeUtf8, false),
Field::new("name", DataType::LargeUtf8, false),
Field::new("role", DataType::LargeUtf8, false),
Field::new("description", DataType::LargeUtf8, false),
Field::new(
"stats",
DataType::Struct(arrow_schema::Fields::from(vec![
Arc::new(Field::new("strength", DataType::Int8, false)),
Arc::new(Field::new("magic", DataType::Int8, false)),
Arc::new(Field::new("leadership", DataType::Int8, false)),
Arc::new(Field::new("wisdom", DataType::Int8, false)),
])),
false,
),
Field::new(
"vector",
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4),
false,
),
]))
}
// --8<-- [end:quickstart_define_struct]
type BatchIter = Box<dyn arrow_array::RecordBatchReader + Send>;
fn characters_to_reader(schema: Arc<Schema>, rows: &[Character]) -> BatchIter {
let ids = LargeStringArray::from_iter_values(rows.iter().map(|row| row.id.as_str()));
let names = LargeStringArray::from_iter_values(rows.iter().map(|row| row.name.as_str()));
let roles = LargeStringArray::from_iter_values(rows.iter().map(|row| row.role.as_str()));
let descriptions =
LargeStringArray::from_iter_values(rows.iter().map(|row| row.description.as_str()));
let strength = Int8Array::from_iter_values(rows.iter().map(|row| row.stats.strength));
let magic = Int8Array::from_iter_values(rows.iter().map(|row| row.stats.magic));
let leadership = Int8Array::from_iter_values(rows.iter().map(|row| row.stats.leadership));
let wisdom = Int8Array::from_iter_values(rows.iter().map(|row| row.stats.wisdom));
let stats_fields: Vec<FieldRef> = vec![
Arc::new(Field::new("strength", DataType::Int8, false)),
Arc::new(Field::new("magic", DataType::Int8, false)),
Arc::new(Field::new("leadership", DataType::Int8, false)),
Arc::new(Field::new("wisdom", DataType::Int8, false)),
];
let stats = StructArray::new(
stats_fields.into(),
vec![
Arc::new(strength),
Arc::new(magic),
Arc::new(leadership),
Arc::new(wisdom),
],
None,
);
let vectors = FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
rows.iter()
.map(|row| Some(row.vector.iter().copied().map(Some).collect::<Vec<_>>())),
4,
);
let batch = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(ids),
Arc::new(names),
Arc::new(roles),
Arc::new(descriptions),
Arc::new(stats),
Arc::new(vectors),
],
)
.unwrap();
Box::new(RecordBatchIterator::new(
vec![Ok(batch)].into_iter(),
schema,
))
}
#[tokio::main]
async fn main() {
let temp_dir = tempfile::tempdir().unwrap();
let uri = temp_dir.path().to_str().unwrap();
let db = connect(uri).execute().await.unwrap();
// --8<-- [start:quickstart_data]
let data = vec![
Character {
id: "1".to_string(),
name: "King Arthur".to_string(),
role: "King".to_string(),
description: "Leader of Camelot and wielder of Excalibur.".to_string(),
stats: Stats {
strength: 4,
magic: 1,
leadership: 5,
wisdom: 4,
},
vector: [0.7, 0.1, 0.9, 0.7],
},
Character {
id: "2".to_string(),
name: "Merlin".to_string(),
role: "Wizard".to_string(),
description: "Advisor and prophet with deep magical knowledge.".to_string(),
stats: Stats {
strength: 2,
magic: 5,
leadership: 4,
wisdom: 5,
},
vector: [0.2, 0.9, 0.4, 0.9],
},
Character {
id: "3".to_string(),
name: "Sir Lancelot".to_string(),
role: "Knight".to_string(),
description: "Legendary knight known for courage and combat skill.".to_string(),
stats: Stats {
strength: 5,
magic: 1,
leadership: 3,
wisdom: 3,
},
vector: [0.9, 0.1, 0.5, 0.4],
},
];
// --8<-- [end:quickstart_data]
// --8<-- [start:quickstart_create_table]
let schema = characters_schema();
let table = db
.create_table("characters", characters_to_reader(schema.clone(), &data))
.mode(CreateTableMode::Overwrite)
.execute()
.await
.unwrap();
// --8<-- [end:quickstart_create_table]
assert_eq!(table.count_rows(None).await.unwrap(), 3);
db.drop_table("characters", &[]).await.unwrap();
// --8<-- [start:quickstart_create_table_no_overwrite]
let table = db
.create_table("characters", characters_to_reader(schema.clone(), &data))
.execute()
.await
.unwrap();
// --8<-- [end:quickstart_create_table_no_overwrite]
assert_eq!(table.count_rows(None).await.unwrap(), 3);
// --8<-- [start:quickstart_vector_search_1]
// Search for examples similar to a "wise magical advisor"
let query_vector = [0.2, 0.8, 0.4, 0.9];
let result: DataFrame = table
.query()
.nearest_to(&query_vector)
.unwrap()
.select(Select::Columns(vec![
"name".to_string(),
"role".to_string(),
"description".to_string(),
"_distance".to_string(),
]))
.limit(2)
.execute()
.await
.unwrap()
.into_polars()
.await
.unwrap();
println!("{result:?}");
// --8<-- [end:quickstart_vector_search_1]
let name_col = result.column("name").unwrap().str().unwrap();
assert_eq!(name_col.get(0).unwrap(), "Merlin");
// --8<-- [start:quickstart_curate_with_metadata]
let curated: DataFrame = table
.query()
.nearest_to(&query_vector)
.unwrap()
.only_if("stats.magic >= 4")
.select(Select::Columns(vec![
"name".to_string(),
"role".to_string(),
"description".to_string(),
"_distance".to_string(),
]))
.limit(2)
.execute()
.await
.unwrap()
.into_polars()
.await
.unwrap();
println!("{curated:?}");
// --8<-- [end:quickstart_curate_with_metadata]
let curated_name_col = curated.column("name").unwrap().str().unwrap();
assert_eq!(curated_name_col.get(0).unwrap(), "Merlin");
// --8<-- [start:quickstart_output_array]
let result: DataFrame = table
.query()
.nearest_to(&query_vector)
.unwrap()
.select(Select::Columns(vec![
"name".to_string(),
"role".to_string(),
"description".to_string(),
"_distance".to_string(),
]))
.limit(2)
.execute()
.await
.unwrap()
.into_polars()
.await
.unwrap();
println!("{result:?}");
// --8<-- [end:quickstart_output_array]
let name_col = result.column("name").unwrap().str().unwrap();
assert_eq!(name_col.get(0).unwrap(), "Merlin");
// --8<-- [start:quickstart_add_feature]
table
.add_columns(
NewColumnTransform::SqlExpressions(vec![(
"power_score".to_string(),
"cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)"
.to_string(),
)]),
None,
)
.await
.unwrap();
// --8<-- [end:quickstart_add_feature]
// --8<-- [start:quickstart_query_feature]
let features: DataFrame = table
.query()
.select(Select::Columns(vec![
"name".to_string(),
"role".to_string(),
"power_score".to_string(),
]))
.execute()
.await
.unwrap()
.into_polars()
.await
.unwrap();
println!("{features:?}");
// --8<-- [end:quickstart_query_feature]
assert!(features.column("power_score").is_ok());
// --8<-- [start:quickstart_multimodal_bytes]
use std::sync::Arc;
use arrow_array::{
BinaryArray, FixedSizeListArray, LargeStringArray, RecordBatch, RecordBatchIterator,
};
use arrow_schema::{DataType, Field, Schema};
let image_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../../web/static/assets/images/quickstart/sir-lancelot.jpg");
let image_bytes = std::fs::read(image_path).unwrap();
let image_schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::LargeUtf8, false),
Field::new("description", DataType::LargeUtf8, false),
Field::new("image", DataType::Binary, false),
Field::new(
"vector",
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4),
false,
),
]));
let image_vectors = [[0.9_f32, 0.1, 0.5, 0.4]];
let image_batch = RecordBatch::try_new(
image_schema.clone(),
vec![
Arc::new(LargeStringArray::from_iter_values(["lancelot"])),
Arc::new(LargeStringArray::from_iter_values([
"Portrait of Sir Lancelot",
])),
Arc::new(BinaryArray::from_iter_values([image_bytes.as_slice()])),
Arc::new(
FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
image_vectors
.iter()
.map(|vector| Some(vector.iter().copied().map(Some).collect::<Vec<_>>())),
4,
),
),
],
)
.unwrap();
let image_reader: Box<dyn arrow_array::RecordBatchReader + Send> = Box::new(
RecordBatchIterator::new(vec![Ok(image_batch)].into_iter(), image_schema),
);
let multimodal_table = db
.create_table("character_images", image_reader)
.mode(CreateTableMode::Overwrite)
.execute()
.await
.unwrap();
// --8<-- [end:quickstart_multimodal_bytes]
assert_eq!(multimodal_table.count_rows(None).await.unwrap(), 1);
}
docs/web-tests/rs/search.rs
+311
View File
@@ -0,0 +1,311 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
//! This example demonstrates vector search query options.
//!
//! Snippets from this example are used in the documentation on vector search.
use std::sync::Arc;
use arrow_array::types::Float32Type;
use arrow_array::{
Array, FixedSizeListArray, Int32Array, Int64Array, RecordBatch, RecordBatchIterator,
RecordBatchReader, UInt8Array,
};
use arrow_schema::{DataType, Field, Schema};
use futures::TryStreamExt;
use lancedb::connection::Connection;
use lancedb::index::vector::IvfFlatIndexBuilder;
use lancedb::index::Index;
use lancedb::query::{ExecutableQuery, QueryBase, Select};
use lancedb::{connect, DistanceType, Result, Table};
const DIM: usize = 128;
#[tokio::main]
async fn main() -> Result<()> {
if std::path::Path::new("data").exists() {
std::fs::remove_dir_all("data").unwrap();
}
let uri = "data/sample-lancedb";
let db = connect(uri).execute().await?;
let tbl = create_table(&db).await?;
tbl.create_index(&["vector"], Index::Auto).execute().await?;
configure_distance_metric(&tbl).await?;
exact_vs_approximate(&tbl).await?;
search_distance_range(&tbl).await?;
vector_search_prefilter(&tbl).await?;
vector_search_postfilter(&tbl).await?;
fast_search(&tbl).await?;
brute_force_search(&tbl).await?;
bypass_vector_index(&tbl).await?;
batch_search(&tbl).await?;
binary_search(&db).await?;
Ok(())
}
fn create_some_records() -> Result<Box<dyn RecordBatchReader + Send>> {
const TOTAL: usize = 1000;
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int32, false),
Field::new(
"vector",
DataType::FixedSizeList(
Arc::new(Field::new("item", DataType::Float32, true)),
DIM as i32,
),
true,
),
]));
let batches = RecordBatchIterator::new(
vec![RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Int32Array::from_iter_values(0..TOTAL as i32)),
Arc::new(
FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
(0..TOTAL).map(|_| Some(vec![Some(1.0); DIM])),
DIM as i32,
),
),
],
)
.unwrap()]
.into_iter()
.map(Ok),
schema.clone(),
);
Ok(Box::new(batches))
}
async fn create_table(db: &Connection) -> Result<Table> {
let initial_data: Box<dyn RecordBatchReader + Send> = create_some_records()?;
let tbl = db
.create_table("my_vectors", initial_data)
.execute()
.await
.unwrap();
Ok(tbl)
}
async fn configure_distance_metric(table: &Table) -> Result<()> {
let query_vector = [1.0; DIM];
// --8<-- [start:configure_distance_metric]
// Use the same distance metric the index was trained with.
let mut results = table
.vector_search(&query_vector)?
.distance_type(DistanceType::Cosine)
.limit(10)
.execute()
.await?;
while let Some(batch) = results.try_next().await? {
println!("{:?}", batch);
}
// --8<-- [end:configure_distance_metric]
Ok(())
}
async fn exact_vs_approximate(table: &Table) -> Result<()> {
let query_vector = [1.0; DIM];
// --8<-- [start:exact_vs_approximate]
// Approximate ANN search (fast, distances may come from the index representation)
let mut fast_results = table.vector_search(&query_vector)?.limit(10).execute().await?;
while let Some(batch) = fast_results.try_next().await? {
println!("{:?}", batch);
}
// Rerank a larger candidate set on full vectors for better recall
let mut refined_results = table
.vector_search(&query_vector)?
.limit(10)
.refine_factor(20)
.execute()
.await?;
while let Some(batch) = refined_results.try_next().await? {
println!("{:?}", batch);
}
// --8<-- [end:exact_vs_approximate]
Ok(())
}
async fn search_distance_range(table: &Table) -> Result<()> {
let query_vector = [1.0; DIM];
// --8<-- [start:search_distance_range]
// Only return rows whose distance falls within [0.1, 0.5).
let mut results = table
.vector_search(&query_vector)?
.distance_range(Some(0.1), Some(0.5))
.execute()
.await?;
while let Some(batch) = results.try_next().await? {
println!("{:?}", batch);
}
// --8<-- [end:search_distance_range]
Ok(())
}
async fn vector_search_prefilter(table: &Table) -> Result<()> {
let query_vector = [1.0; DIM];
// --8<-- [start:vector_search_prefilter]
// Prefiltering is the default: the filter is applied before vector search.
let mut results = table
.vector_search(&query_vector)?
.only_if("id > 100")
.select(Select::columns(&["id"]))
.limit(5)
.execute()
.await?;
while let Some(batch) = results.try_next().await? {
println!("{:?}", batch);
}
// --8<-- [end:vector_search_prefilter]
Ok(())
}
async fn vector_search_postfilter(table: &Table) -> Result<()> {
let query_vector = [1.0; DIM];
// --8<-- [start:vector_search_postfilter]
// Apply the filter after vector search by calling postfilter().
let mut results = table
.vector_search(&query_vector)?
.only_if("id > 100")
.postfilter()
.select(Select::columns(&["id"]))
.limit(5)
.execute()
.await?;
while let Some(batch) = results.try_next().await? {
println!("{:?}", batch);
}
// --8<-- [end:vector_search_postfilter]
Ok(())
}
async fn fast_search(table: &Table) -> Result<()> {
let query_vector = [1.0; DIM];
// --8<-- [start:fast_search]
// Skip unindexed data for lower latency.
let mut results = table
.vector_search(&query_vector)?
.fast_search()
.limit(5)
.execute()
.await?;
while let Some(batch) = results.try_next().await? {
println!("{:?}", batch);
}
// --8<-- [end:fast_search]
Ok(())
}
async fn brute_force_search(table: &Table) -> Result<()> {
let query_vector = [1.0; DIM];
// --8<-- [start:brute_force_search]
// A plain vector search returns the top-k closest rows.
let mut results = table.vector_search(&query_vector)?.limit(3).execute().await?;
while let Some(batch) = results.try_next().await? {
println!("{:?}", batch);
}
// --8<-- [end:brute_force_search]
Ok(())
}
async fn bypass_vector_index(table: &Table) -> Result<()> {
let query_vector = [1.0; DIM];
// --8<-- [start:bypass_vector_index]
// Force an exhaustive (flat) scan for exact, ground-truth results.
let mut results = table
.vector_search(&query_vector)?
.bypass_vector_index()
.limit(5)
.execute()
.await?;
while let Some(batch) = results.try_next().await? {
println!("{:?}", batch);
}
// --8<-- [end:bypass_vector_index]
Ok(())
}
async fn batch_search(table: &Table) -> Result<()> {
let query_1 = [1.0; DIM];
let query_2 = [0.5; DIM];
// --8<-- [start:batch_search]
// Search multiple query vectors in one call. Each result row carries a
// `query_index` mapping it back to the query it matched.
let mut results = table
.vector_search(&query_1)?
.add_query_vector(&query_2)?
.limit(5)
.execute()
.await?;
while let Some(batch) = results.try_next().await? {
println!("{:?}", batch);
}
// --8<-- [end:batch_search]
Ok(())
}
async fn binary_search(db: &Connection) -> Result<()> {
// A 256-bit binary vector is stored as 256 / 8 = 32 packed uint8 bytes.
const NUM_BYTES: i32 = 32;
const TOTAL: usize = 1024;
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int64, false),
Field::new(
"vector",
DataType::FixedSizeList(
Arc::new(Field::new("item", DataType::UInt8, true)),
NUM_BYTES,
),
true,
),
]));
let values =
UInt8Array::from_iter_values((0..TOTAL * NUM_BYTES as usize).map(|i| (i % 256) as u8));
let vectors = FixedSizeListArray::try_new(
Arc::new(Field::new("item", DataType::UInt8, true)),
NUM_BYTES,
Arc::new(values),
None,
)
.unwrap();
let batch = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Int64Array::from_iter_values(0..TOTAL as i64)),
Arc::new(vectors),
],
)
.unwrap();
let reader: Box<dyn RecordBatchReader + Send> =
Box::new(RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema.clone()));
let tbl = db.create_table("binary_vectors", reader).execute().await?;
tbl.create_index(
&["vector"],
Index::IvfFlat(IvfFlatIndexBuilder::default().distance_type(DistanceType::Hamming)),
)
.execute()
.await?;
// --8<-- [start:binary_search]
// Binary vectors use `hamming` distance over the packed uint8 bytes.
let query: Arc<dyn Array> = Arc::new(UInt8Array::from(vec![1u8; NUM_BYTES as usize]));
let mut results = tbl
.vector_search(query)?
.distance_type(DistanceType::Hamming)
.limit(10)
.execute()
.await?;
while let Some(batch) = results.try_next().await? {
println!("{:?}", batch);
}
// --8<-- [end:binary_search]
Ok(())
}
docs/web-tests/rs/sentence_transformers.rs
+95
View File
@@ -0,0 +1,95 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::{iter::once, sync::Arc};
use arrow_array::{RecordBatch, RecordBatchIterator, StringArray};
use arrow_schema::{DataType, Field, Schema};
use futures::StreamExt;
use lancedb::{
arrow::IntoArrow,
connect,
embeddings::{
sentence_transformers::SentenceTransformersEmbeddings, EmbeddingDefinition,
EmbeddingFunction,
},
query::{ExecutableQuery, QueryBase},
Result,
};
#[tokio::main]
async fn main() -> Result<()> {
let tempdir = tempfile::tempdir().unwrap();
let tempdir = tempdir.path().to_str().unwrap();
let embedding = SentenceTransformersEmbeddings::builder().build()?;
let embedding = Arc::new(embedding);
let db = connect(tempdir).execute().await?;
db.embedding_registry()
.register("sentence-transformers", embedding.clone())?;
let table = db
.create_table("vectors", make_data())
.add_embedding(EmbeddingDefinition::new(
"facts",
"sentence-transformers",
Some("embeddings"),
))?
.execute()
.await?;
let query = Arc::new(StringArray::from_iter_values(once(
"How many bones are in the human body?",
)));
let query_vector = embedding.compute_query_embeddings(query)?;
let mut results = table
.vector_search(query_vector)?
.limit(1)
.execute()
.await?;
let rb = results.next().await.unwrap()?;
let out = rb
.column_by_name("facts")
.unwrap()
.as_any()
.downcast_ref::<StringArray>()
.unwrap();
let text = out.iter().next().unwrap().unwrap();
println!("Answer: {}", text);
Ok(())
}
fn make_data() -> impl IntoArrow {
let schema = Schema::new(vec![Field::new("facts", DataType::Utf8, false)]);
let facts = StringArray::from_iter_values(vec![
"Albert Einstein was a theoretical physicist.",
"The capital of France is Paris.",
"The Great Wall of China is one of the Seven Wonders of the World.",
"Python is a popular programming language.",
"Mount Everest is the highest mountain in the world.",
"Leonardo da Vinci painted the Mona Lisa.",
"Shakespeare wrote Hamlet.",
"The human body has 206 bones.",
"The speed of light is approximately 299,792 kilometers per second.",
"Water boils at 100 degrees Celsius.",
"The Earth orbits the Sun.",
"The Pyramids of Giza are located in Egypt.",
"Coffee is one of the most popular beverages in the world.",
"Tokyo is the capital city of Japan.",
"Photosynthesis is the process by which plants make their food.",
"The Pacific Ocean is the largest ocean on Earth.",
"Mozart was a prolific composer of classical music.",
"The Internet is a global network of computers.",
"Basketball is a sport played with a ball and a hoop.",
"The first computer virus was created in 1983.",
"Artificial neural networks are inspired by the human brain.",
"Deep learning is a subset of machine learning.",
"IBM's Watson won Jeopardy! in 2011.",
"The first computer programmer was Ada Lovelace.",
"The first chatbot was ELIZA, created in the 1960s.",
]);
let schema = Arc::new(schema);
let rb = RecordBatch::try_new(schema.clone(), vec![Arc::new(facts)]).unwrap();
Box::new(RecordBatchIterator::new(vec![Ok(rb)], schema))
}
docs/web-tests/rs/simple.rs
+147
View File
@@ -0,0 +1,147 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
//! This example demonstrates basic usage of LanceDb.
//!
//! Snippets from this example are used in the quickstart documentation.
use std::sync::Arc;
use arrow_array::types::Float32Type;
use arrow_array::{FixedSizeListArray, Int32Array, RecordBatch, RecordBatchIterator};
use arrow_schema::{DataType, Field, Schema};
use futures::TryStreamExt;
use lancedb::arrow::IntoArrow;
use lancedb::connection::Connection;
use lancedb::index::Index;
use lancedb::query::{ExecutableQuery, QueryBase};
use lancedb::{connect, Result, Table as LanceDbTable};
// --8<-- [start:connect]
#[tokio::main]
async fn main() -> Result<()> {
if std::path::Path::new("data").exists() {
std::fs::remove_dir_all("data").unwrap();
}
// --8<-- [start:connect_uri]
let uri = "data/sample-lancedb";
let db = connect(uri).execute().await?;
// --8<-- [end:connect_uri]
// --8<-- [start:list_names]
println!("{:?}", db.table_names().execute().await?);
// --8<-- [end:list_names]
let tbl = create_table(&db).await?;
create_index(&tbl).await?;
let batches = search(&tbl).await?;
println!("{:?}", batches);
create_empty_table(&db).await.unwrap();
// --8<-- [start:delete]
tbl.delete("id > 24").await.unwrap();
// --8<-- [end:delete]
// --8<-- [start:drop_table]
db.drop_table("my_table").await.unwrap();
// --8<-- [end:drop_table]
Ok(())
}
// --8<-- [end:connect]
#[allow(dead_code)]
async fn open_with_existing_tbl() -> Result<()> {
let uri = "data/sample-lancedb";
let db = connect(uri).execute().await?;
#[allow(unused_variables)]
// --8<-- [start:open_existing_tbl]
let table = db.open_table("my_table").execute().await.unwrap();
// --8<-- [end:open_existing_tbl]
Ok(())
}
fn create_some_records() -> Result<impl IntoArrow> {
const TOTAL: usize = 1000;
const DIM: usize = 128;
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int32, false),
Field::new(
"vector",
DataType::FixedSizeList(
Arc::new(Field::new("item", DataType::Float32, true)),
DIM as i32,
),
true,
),
]));
// Create a RecordBatch stream.
let batches = RecordBatchIterator::new(
vec![RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Int32Array::from_iter_values(0..TOTAL as i32)),
Arc::new(
FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
(0..TOTAL).map(|_| Some(vec![Some(1.0); DIM])),
DIM as i32,
),
),
],
)
.unwrap()]
.into_iter()
.map(Ok),
schema.clone(),
);
Ok(Box::new(batches))
}
async fn create_table(db: &Connection) -> Result<LanceDbTable> {
// --8<-- [start:create_table]
let initial_data = create_some_records()?;
let tbl = db
.create_table("my_table", initial_data)
.execute()
.await
.unwrap();
// --8<-- [end:create_table]
// --8<-- [start:add]
let new_data = create_some_records()?;
tbl.add(new_data).execute().await.unwrap();
// --8<-- [end:add]
Ok(tbl)
}
async fn create_empty_table(db: &Connection) -> Result<LanceDbTable> {
// --8<-- [start:create_empty_table]
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int32, false),
Field::new("item", DataType::Utf8, true),
]));
db.create_empty_table("empty_table", schema).execute().await
// --8<-- [end:create_empty_table]
}
async fn create_index(table: &LanceDbTable) -> Result<()> {
// --8<-- [start:create_index]
table.create_index(&["vector"], Index::Auto).execute().await
// --8<-- [end:create_index]
}
async fn search(table: &LanceDbTable) -> Result<Vec<RecordBatch>> {
// --8<-- [start:search]
table
.query()
.limit(2)
.nearest_to(&[1.0; 128])?
.execute()
.await?
.try_collect::<Vec<_>>()
.await
// --8<-- [end:search]
}
docs/web-tests/rs/tables.rs
+1516
View File
File diff suppressed because it is too large Load Diff
docs/web-tests/ts/ann_indexes.test.ts
+57
View File
@@ -0,0 +1,57 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { expect, test } from "@jest/globals";
// --8<-- [start:import]
import * as lancedb from "@lancedb/lancedb";
import type { VectorQuery } from "@lancedb/lancedb";
// --8<-- [end:import]
import { withTempDirectory } from "./util.ts";
test("ann index examples", async () => {
await withTempDirectory(async (databaseDir) => {
// --8<-- [start:ingest]
const db = await lancedb.connect(databaseDir);
const data = Array.from({ length: 5_000 }, (_, i) => ({
vector: Array(128).fill(i),
id: `${i}`,
content: "",
longId: `${i}`,
}));
const table = await db.createTable("my_vectors", data, {
mode: "overwrite",
});
await table.createIndex("vector", {
config: lancedb.Index.ivfPq({
numPartitions: 10,
numSubVectors: 16,
}),
});
// --8<-- [end:ingest]
// --8<-- [start:search1]
const search = table.search(Array(128).fill(1.2)).limit(2) as VectorQuery;
const results1 = await search.nprobes(20).refineFactor(10).toArray();
// --8<-- [end:search1]
expect(results1.length).toBe(2);
// --8<-- [start:search2]
const results2 = await table
.search(Array(128).fill(1.2))
.where("id != '1141'")
.limit(2)
.toArray();
// --8<-- [end:search2]
expect(results2.length).toBe(2);
// --8<-- [start:search3]
const results3 = await table
.search(Array(128).fill(1.2))
.select(["id"])
.limit(2)
.toArray();
// --8<-- [end:search3]
expect(results3.length).toBe(2);
});
}, 100_000);
docs/web-tests/ts/basic_usage.test.ts
+161
View File
@@ -0,0 +1,161 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { expect, test } from "@jest/globals";
import * as fs from "node:fs";
import * as path from "node:path";
// --8<-- [start:basic_imports]
import * as lancedb from "@lancedb/lancedb";
import * as arrow from "apache-arrow";
// --8<-- [end:basic_imports]
import { withTempDirectory } from "./util.ts";
const dataPath = path.join(__dirname, "..", "camelot.json");
test("basic usage examples (async)", async () => {
await withTempDirectory(async (databaseDir) => {
const db = await lancedb.connect(databaseDir);
// --8<-- [start:data_load]
const data = JSON.parse(fs.readFileSync(dataPath, "utf-8"));
// --8<-- [end:data_load]
// --8<-- [start:basic_create_table]
let table = await db.createTable("camelot", data, {
mode: "overwrite",
});
// --8<-- [end:basic_create_table]
expect(await table.countRows()).toBe(8);
// --8<-- [start:basic_open_table]
table = await db.openTable("camelot");
// --8<-- [end:basic_open_table]
// --8<-- [start:basic_create_empty_table]
const schema = new arrow.Schema([
new arrow.Field("id", new arrow.Int16()),
new arrow.Field("name", new arrow.Utf8()),
new arrow.Field("role", new arrow.Utf8()),
new arrow.Field("description", new arrow.Utf8()),
new arrow.Field(
"vector",
new arrow.FixedSizeList(
4,
new arrow.Field("item", new arrow.Float32(), true),
),
),
new arrow.Field(
"stats",
new arrow.Struct([
new arrow.Field("strength", new arrow.Int8()),
new arrow.Field("courage", new arrow.Int8()),
new arrow.Field("magic", new arrow.Int8()),
new arrow.Field("wisdom", new arrow.Int8()),
]),
),
]);
await db.createEmptyTable("camelot_empty", schema, { mode: "overwrite" });
// --8<-- [end:basic_create_empty_table]
expect(await db.tableNames()).toContain("camelot_empty");
await db.dropTable("camelot_empty");
// --8<-- [start:basic_add_data]
const magicalCharacters = [
{
id: 9,
name: "Morgan le Fay",
role: "Sorceress",
description:
"A powerful enchantress, Arthur's half-sister, and a complex figure who oscillates between aiding and opposing Camelot.",
vector: [0.1, 0.84, 0.25, 0.7],
stats: { strength: 2, courage: 3, magic: 5, wisdom: 4 },
},
{
id: 10,
name: "The Lady of the Lake",
role: "Mystical Guardian",
description:
"A mysterious supernatural figure associated with Avalon, known for giving Arthur the sword Excalibur.",
vector: [0.0, 0.9, 0.58, 0.88],
stats: { strength: 2, courage: 3, magic: 5, wisdom: 5 },
},
];
await table.add(magicalCharacters);
// --8<-- [end:basic_add_data]
expect(await table.countRows()).toBe(10);
// --8<-- [start:basic_vector_search]
const queryVector = [0.03, 0.85, 0.61, 0.9];
const result = await table.search(queryVector).limit(5).toArray();
console.log(result);
// --8<-- [end:basic_vector_search]
// --8<-- [start:basic_add_columns]
await table.addColumns([
{
name: "power",
valueSql:
"cast(((stats.strength + stats.courage + stats.magic + stats.wisdom) / 4.0) as float)",
},
]);
// --8<-- [end:basic_add_columns]
const schemaWithPower = await table.schema();
expect(schemaWithPower.fields.some((f) => f.name === "power")).toBe(true);
// --8<-- [start:basic_vector_search_q1]
// Who are the characters similar to "wizard"?
const queryVector1 = [0.03, 0.85, 0.61, 0.9];
const r1 = await table
.search(queryVector1)
.limit(5)
.select(["name", "role", "description"])
.toArray();
console.log(r1);
// --8<-- [end:basic_vector_search_q1]
// --8<-- [start:basic_vector_search_q2]
// Who are the characters similar to "wizard" with high magic stats?
const queryVector2 = [0.03, 0.85, 0.61, 0.9];
const r2 = await table
.search(queryVector2)
.where("stats.magic > 3")
.select(["name", "role", "description"])
.limit(5)
.toArray();
console.log(r2);
// --8<-- [end:basic_vector_search_q2]
// --8<-- [start:basic_vector_search_q3]
// Who are the strongest characters?
const r3 = await table
.query()
.where("stats.strength > 3")
.select(["name", "role", "description"])
.limit(5)
.toArray();
console.log(r3);
// --8<-- [end:basic_vector_search_q3]
// --8<-- [start:basic_vector_search_q4]
// Who are the strongest characters?
const r4 = await table
.query()
.select(["name", "role", "description", "power"])
.toArray();
console.log(r4);
// --8<-- [end:basic_vector_search_q4]
// --8<-- [start:basic_drop_columns]
await table.dropColumns(["power"]);
// --8<-- [end:basic_drop_columns]
// --8<-- [start:basic_delete_rows]
await table.delete('role = "Traitor Knight"');
// --8<-- [end:basic_delete_rows]
expect(await table.countRows()).toBe(9);
// --8<-- [start:basic_drop_table]
await db.dropTable("camelot");
// --8<-- [end:basic_drop_table]
expect(await db.tableNames()).not.toContain("camelot");
});
});
docs/web-tests/ts/biome.json
+52
View File
@@ -0,0 +1,52 @@
{
"$schema": "https://biomejs.dev/schemas/1.9.4/schema.json",
"vcs": {
"enabled": false,
"clientKind": "git",
"useIgnoreFile": false
},
"files": {
"ignoreUnknown": false,
"ignore": []
},
"formatter": {
"enabled": true,
"indentStyle": "space"
},
"organizeImports": {
"enabled": true
},
"linter": {
"enabled": true,
"rules": {
"recommended": true
}
},
"javascript": {
"formatter": {
"quoteStyle": "double"
}
},
"overrides": [
{
"include": ["*"],
"linter": {
"rules": {
"style": {
"noNonNullAssertion": "off"
}
}
}
},
{
"include": ["merge_insert.test.ts"],
"linter": {
"rules": {
"style": {
"useNamingConvention": "off"
}
}
}
}
]
}
docs/web-tests/ts/connection.test.ts
+121
View File
@@ -0,0 +1,121 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { expect, jest, test } from "@jest/globals";
import * as path from "node:path";
import { withTempDirectory } from "./util.ts";
// --8<-- [start:connect]
import * as lancedb from "@lancedb/lancedb";
async function connectExample(uri: string) {
const db = await lancedb.connect(uri);
return db;
}
// --8<-- [end:connect]
test("connect to a local database", async () => {
await withTempDirectory(async (tempDir) => {
const uri = path.join(tempDir, "ex_lancedb");
const db = await connectExample(uri);
expect(db).toBeDefined();
});
});
async function connectEnterpriseQuickstart() {
// --8<-- [start:connect_enterprise_quickstart]
const uri = "db://your-database-uri";
const apiKey = "your-api-key";
const region = "us-east-1";
const hostOverride = "https://your-enterprise-endpoint.com";
const db = await lancedb.connect(uri, {
apiKey,
region,
hostOverride,
});
// --8<-- [end:connect_enterprise_quickstart]
return db;
}
test("enterprise quickstart connect uses placeholder config", async () => {
const mockDb = { __mock: true } as unknown as Awaited<
ReturnType<typeof lancedb.connect>
>;
const spy = jest.spyOn(lancedb, "connect").mockResolvedValue(mockDb);
const db = await connectEnterpriseQuickstart();
expect(db).toBe(mockDb);
expect(spy).toHaveBeenCalledWith("db://your-database-uri", {
apiKey: "your-api-key",
region: "us-east-1",
hostOverride: "https://your-enterprise-endpoint.com",
});
spy.mockRestore();
});
// --8<-- [start:connect_object_storage]
async function connectObjectStorageExample() {
const uri = "s3://your-bucket/path";
// You can also use "gs://your-bucket/path" or "az://your-container/path".
const db = await lancedb.connect(uri);
return db;
}
// --8<-- [end:connect_object_storage]
async function namespaceTableOpsExample() {
// --8<-- [start:namespace_table_ops]
const db = await lancedb.connectNamespace("dir", { root: "./local_lancedb" });
// Create namespace tree: prod/search and prod/recommendations
await db.createNamespace(["prod"], { mode: "exist_ok" });
await db.createNamespace(["prod", "search"], { mode: "exist_ok" });
await db.createNamespace(["prod", "recommendations"], { mode: "exist_ok" });
await db.createTable(
"user",
[{ id: 1, vector: [0.1, 0.2], name: "alice" }],
["prod", "search"],
{ mode: "create" }, // use "overwrite" only if you want to replace existing table
);
await db.createTable(
"user",
[{ id: 2, vector: [0.3, 0.4], name: "bob" }],
["prod", "recommendations"],
{ mode: "create" },
);
// Verify
console.log((await db.listNamespaces()).namespaces); // ["prod"]
console.log((await db.listNamespaces(["prod"])).namespaces); // ["recommendations", "search"]
console.log(await db.tableNames(["prod", "search"])); // ["user"]
console.log(await db.tableNames(["prod", "recommendations"])); // ["user"]
// --8<-- [end:namespace_table_ops]
}
async function namespaceAdminOpsExample() {
// --8<-- [start:namespace_admin_ops]
const db = await lancedb.connectNamespace("dir", { root: "./local_lancedb" });
const namespace = ["prod", "search"];
await db.createNamespace(["prod"]);
await db.createNamespace(["prod", "search"]);
const childNamespaces = (await db.listNamespaces(["prod"])).namespaces;
console.log(`Child namespaces under ${JSON.stringify(namespace)}:`, childNamespaces);
// Child namespaces under ["prod","search"]: [ 'search' ]
const metadata = await db.describeNamespace(["prod", "search"]);
console.log(`Metadata for namespace ${JSON.stringify(namespace)}:`, metadata);
await db.dropNamespace(["prod", "search"], { mode: "skip" });
await db.dropNamespace(["prod"], { mode: "skip" });
// --8<-- [end:namespace_admin_ops]
return { childNamespaces, metadata };
}
void [
connectObjectStorageExample,
connectEnterpriseQuickstart,
namespaceTableOpsExample,
namespaceAdminOpsExample,
];
docs/web-tests/ts/custom_embedding_function.test.ts
+79
View File
@@ -0,0 +1,79 @@
import {
type FeatureExtractionPipeline,
pipeline,
} from "@huggingface/transformers";
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { expect, test } from "@jest/globals";
// --8<-- [start:imports]
import * as lancedb from "@lancedb/lancedb";
import {
LanceSchema,
TextEmbeddingFunction,
getRegistry,
register,
} from "@lancedb/lancedb/embedding";
// --8<-- [end:imports]
import { withTempDirectory } from "./util.ts";
// --8<-- [start:embedding_impl]
@register("sentence-transformers")
class SentenceTransformersEmbeddings extends TextEmbeddingFunction {
name = "Xenova/all-miniLM-L6-v2";
#ndims!: number;
extractor!: FeatureExtractionPipeline;
async init() {
this.extractor = await pipeline("feature-extraction", this.name, {
dtype: "fp32",
});
this.#ndims = await this.generateEmbeddings(["hello"]).then(
(e) => e[0].length,
);
}
ndims() {
return this.#ndims;
}
toJSON() {
return {
name: this.name,
};
}
async generateEmbeddings(texts: string[]) {
const output = await this.extractor(texts, {
pooling: "mean",
normalize: true,
});
return output.tolist();
}
}
// --8<-- [end:embedding_impl]
test("Registry examples", async () => {
await withTempDirectory(async (databaseDir) => {
// --8<-- [start:call_custom_function]
const registry = getRegistry();
const sentenceTransformer = await registry
.get<SentenceTransformersEmbeddings>("sentence-transformers")!
.create();
const schema = LanceSchema({
vector: sentenceTransformer.vectorField(),
text: sentenceTransformer.sourceField(),
});
const db = await lancedb.connect(databaseDir);
const table = await db.createEmptyTable("table", schema, {
mode: "overwrite",
});
await table.add([{ text: "hello" }, { text: "world" }]);
const results = await table.search("greeting").limit(1).toArray();
// --8<-- [end:call_custom_function]
expect(results.length).toBe(1);
});
}, 100_000);
docs/web-tests/ts/embedding.test.ts
+160
View File
@@ -0,0 +1,160 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { expect, test } from "@jest/globals";
// --8<-- [start:imports]
import * as lancedb from "@lancedb/lancedb";
import "@lancedb/lancedb/embedding/openai";
import { LanceSchema, getRegistry, register } from "@lancedb/lancedb/embedding";
import { EmbeddingFunction } from "@lancedb/lancedb/embedding";
import { type Float, Float32, Utf8 } from "apache-arrow";
// --8<-- [end:imports]
import { withTempDirectory } from "./util.ts";
const openAiTest = process.env.OPENAI_API_KEY == null ? test.skip : test;
openAiTest("create embedding function", () => {
// --8<-- [start:create_embedding_function]
const func = getRegistry().get("openai")!.create({
model: "text-embedding-3-small",
});
// --8<-- [end:create_embedding_function]
expect(func).toBeDefined();
});
openAiTest("openai embeddings", async () => {
await withTempDirectory(async (databaseDir) => {
// --8<-- [start:openai_embeddings]
const db = await lancedb.connect(databaseDir);
const func = getRegistry()
.get("openai")
?.create({ model: "text-embedding-ada-002" }) as EmbeddingFunction;
const wordsSchema = LanceSchema({
text: func.sourceField(new Utf8()),
vector: func.vectorField(),
});
const tbl = await db.createEmptyTable("words", wordsSchema, {
mode: "overwrite",
});
await tbl.add([{ text: "hello world" }, { text: "goodbye world" }]);
const query = "greetings";
const actual = (await tbl.search(query).limit(1).toArray())[0];
// --8<-- [end:openai_embeddings]
expect(actual).toHaveProperty("text");
});
});
openAiTest("manual query embeddings", async () => {
await withTempDirectory(async (databaseDir) => {
// --8<-- [start:manual_query_embeddings]
const db = await lancedb.connect(databaseDir);
const func = getRegistry()
.get("openai")
?.create({ model: "text-embedding-ada-002" }) as EmbeddingFunction;
const wordsSchema = LanceSchema({
text: func.sourceField(new Utf8()),
vector: func.vectorField(),
});
const tbl = await db.createEmptyTable("words", wordsSchema, {
mode: "overwrite",
});
await tbl.add([{ text: "hello world" }, { text: "goodbye world" }]);
const queryVector = await func.computeQueryEmbeddings("greetings");
// --8<-- [start:manual_query_search]
// queryVector is assumed to already be generated by your embedding function
const actual = (await tbl.search(queryVector).limit(1).toArray())[0];
// --8<-- [end:manual_query_search]
// --8<-- [end:manual_query_embeddings]
expect(actual).toHaveProperty("text");
});
});
test("custom embedding function", async () => {
await withTempDirectory(async (databaseDir) => {
// --8<-- [start:embedding_function]
const db = await lancedb.connect(databaseDir);
@register("my_embedding")
class MyEmbeddingFunction extends EmbeddingFunction<string> {
constructor(optionsRaw = {}) {
super();
const options = this.resolveVariables(optionsRaw);
// Initialize using options
}
ndims() {
return 3;
}
protected getSensitiveKeys(): string[] {
return [];
}
embeddingDataType(): Float {
return new Float32();
}
async computeQueryEmbeddings(_data: string) {
// This is a placeholder for a real embedding function
return [1, 2, 3];
}
async computeSourceEmbeddings(data: string[]) {
// This is a placeholder for a real embedding function
return Array.from({ length: data.length }).fill([
1, 2, 3,
]) as number[][];
}
}
const func = new MyEmbeddingFunction();
const data = [{ text: "pepperoni" }, { text: "pineapple" }];
// Option 1: manually specify the embedding function
const table = await db.createTable("vectors", data, {
embeddingFunction: {
function: func,
sourceColumn: "text",
vectorColumn: "vector",
},
mode: "overwrite",
});
// Option 2: provide the embedding function through a schema
const schema = LanceSchema({
text: func.sourceField(new Utf8()),
vector: func.vectorField(),
});
const table2 = await db.createTable("vectors2", data, {
schema,
mode: "overwrite",
});
// --8<-- [end:embedding_function]
expect(await table.countRows()).toBe(2);
expect(await table2.countRows()).toBe(2);
});
});
test("embedding function api_key", async () => {
// --8<-- [start:register_secret]
const registry = getRegistry();
registry.setVar("api_key", "sk-...");
const func = registry.get("openai")!.create({
apiKey: "$var:api_key",
});
// --8<-- [end:register_secret]
});
openAiTest("embedding function variable fallback", () => {
// --8<-- [start:register_model_fallback]
const registry = getRegistry();
registry.setVar("openai_model", "text-embedding-3-large");
const func = registry.get("openai")!.create({
model: "$var:openai_model:text-embedding-3-small",
});
// --8<-- [end:register_model_fallback]
expect(func).toBeDefined();
});
docs/web-tests/ts/filtering.test.ts
+42
View File
@@ -0,0 +1,42 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { expect, test } from "@jest/globals";
import * as lancedb from "@lancedb/lancedb";
import { withTempDirectory } from "./util.ts";
test("filtering examples", async () => {
await withTempDirectory(async (databaseDir) => {
const db = await lancedb.connect(databaseDir);
const data = Array.from({ length: 10_000 }, (_, i) => ({
vector: Array(1536).fill(i),
id: i,
item: `item ${i}`,
strId: `${i}`,
}));
const tbl = await db.createTable("myVectors", data, { mode: "overwrite" });
// --8<-- [start:search]
const _result = await tbl
.search(Array(1536).fill(0.5))
.limit(1)
.where("id = 10")
.toArray();
// --8<-- [end:search]
// --8<-- [start:vec_search]
const result = await (
tbl.search(Array(1536).fill(0)) as lancedb.VectorQuery
)
.where("(item IN ('item 0', 'item 2')) AND (id > 10)")
.postfilter()
.toArray();
// --8<-- [end:vec_search]
expect(result.length).toBe(0);
// --8<-- [start:sql_search]
await tbl.query().where("id = 10").limit(10).toArray();
// --8<-- [end:sql_search]
});
});
docs/web-tests/ts/full_text_search.test.ts
+45
View File
@@ -0,0 +1,45 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { expect, test } from "@jest/globals";
import * as lancedb from "@lancedb/lancedb";
import { withTempDirectory } from "./util.ts";
test("full text search", async () => {
await withTempDirectory(async (databaseDir) => {
const db = await lancedb.connect(databaseDir);
const words = [
"apple",
"banana",
"cherry",
"date",
"elderberry",
"fig",
"grape",
];
const data = Array.from({ length: 10_000 }, (_, i) => ({
vector: Array(1536).fill(i),
id: i,
item: `item ${i}`,
strId: `${i}`,
doc: words[i % words.length],
}));
const tbl = await db.createTable("myVectors", data, { mode: "overwrite" });
await tbl.createIndex("doc", {
config: lancedb.Index.fts(),
});
// --8<-- [start:full_text_search]
const result = await tbl
.query()
.nearestToText("apple")
.select(["id", "doc"])
.limit(10)
.toArray();
expect(result.length).toBe(10);
// --8<-- [end:full_text_search]
});
}, 10_000);
docs/web-tests/ts/integrations.ts
+134
View File
@@ -0,0 +1,134 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// --8<-- [start:frameworks_genkit_usage]
import { lancedbIndexerRef, lancedb, lancedbRetrieverRef, WriteMode } from "genkitx-lancedb";
import { textEmbedding004, vertexAI } from "@genkit-ai/vertexai";
import { gemini } from "@genkit-ai/vertexai";
import { z, genkit } from "genkit";
import { Document } from "genkit/retriever";
import { chunk } from "llm-chunk";
import { readFile } from "fs/promises";
import path from "path";
import pdf from "pdf-parse/lib/pdf-parse";
const ai = genkit({
plugins: [
// vertexAI provides the textEmbedding004 embedder
vertexAI(),
// the local vector store requires an embedder to translate from text to vector
lancedb([
{
dbUri: ".db", // optional lancedb uri, default to .db
tableName: "table", // optional table name, default to table
embedder: textEmbedding004,
},
]),
],
});
// --8<-- [end:frameworks_genkit_usage]
// --8<-- [start:frameworks_genkit_custom_indexer]
export const menuPdfIndexer = lancedbIndexerRef({
// Using all defaults, for dbUri, tableName, and embedder, etc
});
const chunkingConfig = {
minLength: 1000,
maxLength: 2000,
splitter: "sentence",
overlap: 100,
delimiters: "",
} as any;
async function extractTextFromPdf(filePath: string) {
const pdfFile = path.resolve(filePath);
const dataBuffer = await readFile(pdfFile);
const data = await pdf(dataBuffer);
return data.text;
}
export const indexMenu = ai.defineFlow(
{
name: "indexMenu",
inputSchema: z.string().describe("PDF file path"),
outputSchema: z.void(),
},
async (filePath: string) => {
filePath = path.resolve(filePath);
// Read the pdf.
const pdfTxt = await ai.run("extract-text", () => extractTextFromPdf(filePath));
// Divide the pdf text into segments.
const chunks = await ai.run("chunk-it", async () => chunk(pdfTxt, chunkingConfig));
// Convert chunks of text into documents to store in the index.
const documents = chunks.map((text) => {
return Document.fromText(text, { filePath });
});
// Add documents to the index.
await ai.index({
indexer: menuPdfIndexer,
documents,
options: {
writeMode: WriteMode.Overwrite,
} as any,
});
},
);
// --8<-- [end:frameworks_genkit_custom_indexer]
// --8<-- [start:frameworks_genkit_custom_retriever]
export const menuRetriever = lancedbRetrieverRef({
tableName: "table", // Use the same table name as the indexer.
displayName: "Menu", // Use a custom display name.
});
export const menuQAFlow = ai.defineFlow(
{ name: "Menu", inputSchema: z.string(), outputSchema: z.string() },
async (input: string) => {
// retrieve relevant documents
const docs = await ai.retrieve({
retriever: menuRetriever,
query: input,
options: {
k: 3,
},
});
const extractedContent = docs.map((doc) => {
if (doc.content && Array.isArray(doc.content) && doc.content.length > 0) {
if (doc.content[0].media && doc.content[0].media.url) {
return doc.content[0].media.url;
}
}
return "No content found";
});
console.log("Extracted content:", extractedContent);
const { text } = await ai.generate({
model: gemini("gemini-2.0-flash"),
prompt: `
You are acting as a helpful AI assistant that can answer
questions about the food available on the menu at Genkit Grub Pub.
Use only the context provided to answer the question.
If you don't know, do not make up an answer.
Do not add or change items on the menu.
Context:
${extractedContent.join("\n\n")}
Question: ${input}`,
docs,
});
return text;
},
);
// --8<-- [end:frameworks_genkit_custom_retriever]
docs/web-tests/ts/jest.config.cjs
+6
View File
@@ -0,0 +1,6 @@
/** @type {import('ts-jest').JestConfigWithTsJest} */
module.exports = {
preset: "ts-jest",
testEnvironment: "node",
testPathIgnorePatterns: ["./dist"],
};
docs/web-tests/ts/merge_insert.test.ts
+68
View File
@@ -0,0 +1,68 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { expect, test } from "@jest/globals";
import * as lancedb from "@lancedb/lancedb";
test("basic upsert", async () => {
const db = await lancedb.connect("memory://");
// --8<-- [start:upsert_basic]
const table = await db.createTable("users", [
{ id: 0, name: "Alice" },
{ id: 1, name: "Bob" },
]);
const newUsers = [
{ id: 1, name: "Bobby" },
{ id: 2, name: "Charlie" },
];
await table
.mergeInsert("id")
.whenMatchedUpdateAll()
.whenNotMatchedInsertAll()
.execute(newUsers);
await table.countRows(); // 3
// --8<-- [end:upsert_basic]
expect(await table.countRows()).toBe(3);
// --8<-- [start:insert_if_not_exists]
const table2 = await db.createTable("domains", [
{ domain: "google.com", name: "Google" },
{ domain: "github.com", name: "GitHub" },
]);
const newDomains = [
{ domain: "google.com", name: "Google" },
{ domain: "facebook.com", name: "Facebook" },
];
await table2
.mergeInsert("domain")
.whenNotMatchedInsertAll()
.execute(newDomains);
await table2.countRows(); // 3
// --8<-- [end:insert_if_not_exists]
expect(await table2.countRows()).toBe(3);
// --8<-- [start:replace_range]
const table3 = await db.createTable("chunks", [
{ doc_id: 0, chunk_id: 0, text: "Hello" },
{ doc_id: 0, chunk_id: 1, text: "World" },
{ doc_id: 1, chunk_id: 0, text: "Foo" },
{ doc_id: 1, chunk_id: 1, text: "Bar" },
]);
const newChunks = [{ doc_id: 1, chunk_id: 0, text: "Baz" }];
await table3
.mergeInsert(["doc_id", "chunk_id"])
.whenMatchedUpdateAll()
.whenNotMatchedInsertAll()
.whenNotMatchedBySourceDelete({ where: "doc_id = 1" })
.execute(newChunks);
await table3.countRows("doc_id = 1"); // 1
// --8<-- [end:replace_range]
expect(await table3.countRows("doc_id = 1")).toBe(1);
});
docs/web-tests/ts/multimodal.test.ts
+104
View File
@@ -0,0 +1,104 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { expect, test } from "@jest/globals";
// --8<-- [start:multimodal_imports]
import * as arrow from "apache-arrow";
import { Buffer } from "node:buffer";
import * as lancedb from "@lancedb/lancedb";
// --8<-- [end:multimodal_imports]
import { withTempDirectory } from "./util.ts";
test("multimodal snippets (async)", async () => {
await withTempDirectory(async (databaseDir) => {
const db = await lancedb.connect(databaseDir);
// --8<-- [start:create_dummy_data]
const createDummyImage = (color: string): Uint8Array => {
const pngHeader = Uint8Array.from([137, 80, 78, 71, 13, 10, 26, 10]);
return Buffer.concat([Buffer.from(pngHeader), Buffer.from(color, "utf8")]);
};
const data = [
{
id: 1,
filename: "red_square.png",
vector: Array.from({ length: 128 }, (_, i) => (i % 16) / 16),
image_blob: createDummyImage("red"),
label: "red",
},
{
id: 2,
filename: "blue_square.png",
vector: Array.from({ length: 128 }, (_, i) => ((i + 8) % 16) / 16),
image_blob: createDummyImage("blue"),
label: "blue",
},
];
// --8<-- [end:create_dummy_data]
// --8<-- [start:define_schema]
const schema = new arrow.Schema([
new arrow.Field("id", new arrow.Int32()),
new arrow.Field("filename", new arrow.Utf8()),
new arrow.Field(
"vector",
new arrow.FixedSizeList(
128,
new arrow.Field("item", new arrow.Float32(), true),
),
),
new arrow.Field("image_blob", new arrow.Binary()),
new arrow.Field("label", new arrow.Utf8()),
]);
// --8<-- [end:define_schema]
// --8<-- [start:ingest_data]
const multimodalData = lancedb.makeArrowTable(data, { schema });
const tbl = await db.createTable("images", multimodalData, {
mode: "overwrite",
});
// --8<-- [end:ingest_data]
expect(await tbl.countRows()).toBe(2);
// --8<-- [start:search_data]
const queryVector = Array.from({ length: 128 }, (_, i) => (i % 16) / 16);
const results = await tbl.search(queryVector).limit(1).toArray();
// --8<-- [end:search_data]
// --8<-- [start:process_results]
for (const row of results) {
const imageBytes = row.image_blob as Uint8Array;
console.log(
`Retrieved image: ${row.filename}, Byte length: ${imageBytes.length}`,
);
}
// --8<-- [end:process_results]
expect(results).toHaveLength(1);
// --8<-- [start:blob_api_schema]
const blobSchema = new arrow.Schema([
new arrow.Field("id", new arrow.Int64()),
new arrow.Field(
"video",
new arrow.LargeBinary(),
true,
new Map([["lance-encoding:blob", "true"]]),
),
]);
// --8<-- [end:blob_api_schema]
// --8<-- [start:blob_api_ingest]
const blobData = lancedb.makeArrowTable(
[
{ id: 1, video: Buffer.from("fake_video_bytes_1") },
{ id: 2, video: Buffer.from("fake_video_bytes_2") },
],
{ schema: blobSchema },
);
const blobTable = await db.createTable("videos", blobData, {
mode: "overwrite",
});
// --8<-- [end:blob_api_ingest]
expect(await blobTable.countRows()).toBe(2);
});
});
docs/web-tests/ts/package-lock.json
Generated
+5400
View File
File diff suppressed because it is too large Load Diff
docs/web-tests/ts/package.json
+36
View File
@@ -0,0 +1,36 @@
{
"name": "examples",
"version": "1.0.0",
"description": "Examples for LanceDB",
"main": "index.js",
"type": "module",
"scripts": {
"//1": "--experimental-vm-modules is needed to run jest with sentence-transformers",
"//2": "--testEnvironment is needed to run jest with sentence-transformers",
"//3": "See: https://github.com/huggingface/transformers.js/issues/57",
"test": "node --experimental-vm-modules node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose",
"lint": "biome check *.ts && biome format *.ts",
"lint-ci": "biome ci .",
"lint-fix": "biome check --write *.ts && npm run format",
"format": "biome format --write *.ts"
},
"author": "Lance Devs",
"license": "Apache-2.0",
"dependencies": {
"@huggingface/transformers": "^3.0.2",
"@lancedb/lancedb": "^0.31.0",
"openai": "^4.29.2",
"sharp": "^0.35.5"
},
"devDependencies": {
"@biomejs/biome": "^1.7.3",
"@jest/globals": "^29.7.0",
"jest": "^29.7.0",
"jest-environment-node-single-context": "^29.4.0",
"ts-jest": "^29.2.5",
"typescript": "^5.5.4"
},
"overrides": {
"sharp": "$sharp"
}
}
docs/web-tests/ts/quickstart.test.ts
+176
View File
@@ -0,0 +1,176 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { expect, test } from "@jest/globals";
import * as lancedb from "@lancedb/lancedb";
import { withTempDirectory } from "./util.ts";
test("quickstart example (async)", async () => {
await withTempDirectory(async (databaseDir) => {
const db = await lancedb.connect(databaseDir);
// --8<-- [start:quickstart_data]
const data = [
{
id: "1",
name: "King Arthur",
role: "King",
description: "Leader of Camelot and wielder of Excalibur.",
stats: { strength: 4, magic: 1, leadership: 5, wisdom: 4 },
vector: [0.7, 0.1, 0.9, 0.7],
},
{
id: "2",
name: "Merlin",
role: "Wizard",
description: "Advisor and prophet with deep magical knowledge.",
stats: { strength: 2, magic: 5, leadership: 4, wisdom: 5 },
vector: [0.2, 0.9, 0.4, 0.9],
},
{
id: "3",
name: "Sir Lancelot",
role: "Knight",
description: "Legendary knight known for courage and combat skill.",
stats: { strength: 5, magic: 1, leadership: 3, wisdom: 3 },
vector: [0.9, 0.1, 0.5, 0.4],
},
];
// --8<-- [end:quickstart_data]
// --8<-- [start:quickstart_create_table]
let table = await db.createTable("characters", data, { mode: "overwrite" });
// --8<-- [end:quickstart_create_table]
expect(await table.countRows()).toBe(3);
await db.dropTable("characters");
// --8<-- [start:quickstart_create_table_no_overwrite]
table = await db.createTable("characters", data);
// --8<-- [end:quickstart_create_table_no_overwrite]
expect(await table.countRows()).toBe(3);
// --8<-- [start:quickstart_vector_search_1]
// Search for examples similar to a "wise magical advisor"
let queryVector = [0.2, 0.8, 0.4, 0.9];
let result = await table
.search(queryVector)
.select(["name", "role", "description", "_distance"])
.limit(2)
.toArray();
console.table(result);
// --8<-- [end:quickstart_vector_search_1]
expect(result[0].name).toBe("Merlin");
// --8<-- [start:quickstart_curate_with_metadata]
const curated = await table
.search(queryVector)
.where("stats.magic >= 4")
.select(["name", "role", "description", "_distance"])
.limit(2)
.toArray();
console.table(curated);
// --8<-- [end:quickstart_curate_with_metadata]
expect(curated[0].name).toBe("Merlin");
// --8<-- [start:quickstart_output_array]
result = await table.search(queryVector).limit(2).toArray();
console.table(result);
// --8<-- [end:quickstart_output_array]
expect(result[0].name).toBe("Merlin");
// --8<-- [start:quickstart_add_feature]
await table.addColumns([
{
name: "power_score",
valueSql:
"cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)",
},
]);
// --8<-- [end:quickstart_add_feature]
const schemaWithFeature = await table.schema();
expect(schemaWithFeature.fields.some((f) => f.name === "power_score")).toBe(
true,
);
// --8<-- [start:quickstart_query_feature]
const features = await table
.query()
.select(["name", "role", "power_score"])
.toArray();
console.table(features);
// --8<-- [end:quickstart_query_feature]
expect(features[0]).toHaveProperty("power_score");
// --8<-- [start:quickstart_multimodal_bytes]
const arrow = await import("apache-arrow");
const path = await import("node:path");
const { readFile } = await import("node:fs/promises");
const imagePath = path.join(
__dirname,
"../../web/static/assets/images/quickstart/sir-lancelot.jpg",
);
const imageBytes = await readFile(imagePath);
const imageSchema = new arrow.Schema([
new arrow.Field("id", new arrow.Utf8()),
new arrow.Field("description", new arrow.Utf8()),
new arrow.Field("image", new arrow.Binary()),
new arrow.Field(
"vector",
new arrow.FixedSizeList(
4,
new arrow.Field("item", new arrow.Float32(), true),
),
),
]);
const imageData = lancedb.makeArrowTable(
[
{
id: "lancelot",
description: "Portrait of Sir Lancelot",
image: imageBytes,
vector: [0.9, 0.1, 0.5, 0.4],
},
],
{ schema: imageSchema },
);
const multimodalTable = await db.createTable(
"character_images",
imageData,
{ mode: "overwrite" },
);
// --8<-- [end:quickstart_multimodal_bytes]
expect(await multimodalTable.countRows()).toBe(1);
// --8<-- [start:quickstart_open_table]
table = await db.openTable("characters");
// --8<-- [end:quickstart_open_table]
// --8<-- [start:quickstart_add_data]
const moreData = [
{
id: "4",
name: "Morgana",
role: "Sorceress",
description: "Powerful sorceress of Avalon.",
stats: { strength: 2, magic: 5, leadership: 4, wisdom: 4 },
vector: [0.3, 0.9, 0.6, 0.8],
power_score: 3.75,
},
];
// Add data to table
await table.add(moreData);
// --8<-- [end:quickstart_add_data]
expect(await table.countRows()).toBe(4);
// --8<-- [start:quickstart_vector_search_2]
// Search for examples similar to a "powerful sorceress"
queryVector = [0.3, 0.9, 0.6, 0.8];
const results = await table.search(queryVector).limit(2).toArray();
console.table(results);
// --8<-- [end:quickstart_vector_search_2]
expect(results[0].name).toBe("Morgana");
});
});
docs/web-tests/ts/search.test.ts
+355
View File
@@ -0,0 +1,355 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { expect, test } from "@jest/globals";
// --8<-- [start:import]
import * as lancedb from "@lancedb/lancedb";
// --8<-- [end:import]
// --8<-- [start:import_bin_util]
import { Field, FixedSizeList, Int32, Schema, Uint8 } from "apache-arrow";
// --8<-- [end:import_bin_util]
import { Float32, Struct } from "apache-arrow";
import { withTempDirectory } from "./util.ts";
async function buildIndexedTable(db: lancedb.Connection, name: string) {
const data = Array.from({ length: 512 }, (_, i) => ({
vector: Array.from({ length: 128 }, () => Math.random()),
id: i,
}));
const tbl = await db.createTable(name, data, { mode: "overwrite" });
await tbl.createIndex("vector", {
config: lancedb.Index.ivfPq({ numPartitions: 4, numSubVectors: 8 }),
});
return tbl;
}
test("vector search", async () => {
await withTempDirectory(async (databaseDir) => {
{
const db = await lancedb.connect(databaseDir);
const data = Array.from({ length: 10_000 }, (_, i) => ({
vector: Array(128).fill(i),
id: `${i}`,
}));
await db.createTable("my_vectors", data);
}
// --8<-- [start:search1]
const db = await lancedb.connect(databaseDir);
const tbl = await db.openTable("my_vectors");
const results1 = await tbl.search(Array(128).fill(1.2)).limit(10).toArray();
// --8<-- [end:search1]
expect(results1.length).toBe(10);
// --8<-- [start:search2]
const results2 = await (
tbl.search(Array(128).fill(1.2)) as lancedb.VectorQuery
)
.distanceType("cosine")
.limit(10)
.toArray();
// --8<-- [end:search2]
expect(results2.length).toBe(10);
// --8<-- [start:distance_range]
const results3 = await (
tbl.search(Array(128).fill(1.2)) as lancedb.VectorQuery
)
.distanceType("cosine")
.distanceRange(0.1, 0.2)
.limit(10)
.toArray();
// --8<-- [end:distance_range]
for (const r of results3) {
expect(r.distance).toBeGreaterThanOrEqual(0.1);
expect(r.distance).toBeLessThan(0.2);
}
{
// --8<-- [start:ingest_binary_data]
const schema = new Schema([
new Field("id", new Int32(), true),
new Field("vec", new FixedSizeList(32, new Field("item", new Uint8()))),
]);
const data = lancedb.makeArrowTable(
Array(1_000)
.fill(0)
.map((_, i) => ({
// the 256 bits would be store in 32 bytes,
// if your data is already in this format, you can skip the packBits step
id: i,
vec: lancedb.packBits(Array(256).fill(i % 2)),
})),
{ schema: schema },
);
const tbl = await db.createTable("binary_table", data);
await tbl.createIndex("vec", {
config: lancedb.Index.ivfFlat({
numPartitions: 10,
distanceType: "hamming",
}),
});
// --8<-- [end:ingest_binary_data]
// --8<-- [start:search_binary_data]
const query = Array(32)
.fill(1)
.map(() => Math.floor(Math.random() * 255));
const results = await tbl.query().nearestTo(query).limit(10).toArrow();
// --8<-- [end:search_binary_data]
expect(results.numRows).toBe(10);
}
});
});
test("vector search docs snippets", async () => {
await withTempDirectory(async (databaseDir) => {
const db = await lancedb.connect(databaseDir);
// ---- Nested vector column: inference and explicit selection ----
{
const nestedSchema = new Schema([
new Field("id", new Int32()),
new Field(
"image",
new Struct([
new Field(
"embedding",
new FixedSizeList(2, new Field("item", new Float32(), true)),
),
]),
),
]);
const arrowTbl = lancedb.makeArrowTable(
[{ id: 0, image: { embedding: [0.0, 1.0] } }],
{ schema: nestedSchema },
);
await db.createTable("nested", arrowTbl, { mode: "overwrite" });
// --8<-- [start:select_vector_column]
const table = await db.openTable("nested");
// Inferred: LanceDB finds the single nested vector leaf automatically.
await table.query().nearestTo([0.0, 1.0]).limit(1).toArray();
// Explicit: required when more than one vector column matches.
await table
.query()
.nearestTo([0.0, 1.0])
.column("image.embedding")
.limit(1)
.toArray();
// --8<-- [end:select_vector_column]
}
// ---- Index a nested vector column ----
{
const dim = 16;
const nestedSchema = new Schema([
new Field("id", new Int32()),
new Field(
"image",
new Struct([
new Field(
"embedding",
new FixedSizeList(dim, new Field("item", new Float32(), true)),
),
]),
),
]);
const rows = Array.from({ length: 512 }, (_, i) => ({
id: i,
image: { embedding: Array.from({ length: dim }, () => Math.random()) },
}));
const arrowTbl = lancedb.makeArrowTable(rows, { schema: nestedSchema });
const table = await db.createTable("nested_index", arrowTbl, {
mode: "overwrite",
});
// --8<-- [start:index_nested_column]
await table.createIndex("image.embedding");
// --8<-- [end:index_nested_column]
}
// ---- Indexed queries: refine, fast search, bypass index ----
{
const table = await buildIndexedTable(db, "ann_table");
const embedding = Array.from({ length: 128 }, () => Math.random());
// --8<-- [start:exact_vs_approximate]
// Indexed ANN search without refinement (fast, approximate `_distance`)
const fastResults = await (table.search(embedding) as lancedb.VectorQuery)
.limit(10)
.toArray();
// Recompute distances on full vectors for reranked candidates
const exactDistanceResults = await (
table.search(embedding) as lancedb.VectorQuery
)
.limit(10)
.refineFactor(1)
.toArray();
// Rerank a larger candidate set for better recall (higher latency)
const higherRecallResults = await (
table.search(embedding) as lancedb.VectorQuery
)
.limit(10)
.refineFactor(20)
.toArray();
// --8<-- [end:exact_vs_approximate]
expect(fastResults.length).toBe(10);
expect(exactDistanceResults.length).toBe(10);
expect(higherRecallResults.length).toBe(10);
// --8<-- [start:fast_search]
await table
.query()
.nearestTo(embedding)
.fastSearch()
.limit(5)
.toArray();
// --8<-- [end:fast_search]
// --8<-- [start:bypass_vector_index]
await table
.query()
.nearestTo(embedding)
.bypassVectorIndex()
.limit(5)
.toArray();
// --8<-- [end:bypass_vector_index]
}
// ---- Brute force search (no index) ----
{
const data = Array.from({ length: 64 }, (_, i) => ({
vector: Array(128).fill(i),
id: `${i}`,
}));
await db.createTable("my_vectors", data, { mode: "overwrite" });
// --8<-- [start:brute_force_search]
const tbl = await db.openTable("my_vectors");
const results1 = await tbl.search(Array(128).fill(1.2)).limit(3).toArray();
// --8<-- [end:brute_force_search]
expect(results1.length).toBe(3);
}
// ---- Binary (hamming) vector search ----
{
const schema = new Schema([
new Field("id", new Int32(), true),
new Field("vector", new FixedSizeList(32, new Field("item", new Uint8()))),
]);
const data = lancedb.makeArrowTable(
Array(1000)
.fill(0)
.map((_, i) => ({
// the 256 bits are stored in 32 bytes; if your data is already in
// this format, you can skip the packBits step
id: i,
vector: lancedb.packBits(Array(256).fill(i % 2)),
})),
{ schema },
);
// --8<-- [start:binary_search]
const tbl = await db.createTable("binary_vectors", data, {
mode: "overwrite",
});
await tbl.createIndex("vector", {
config: lancedb.Index.ivfFlat({
numPartitions: 10,
distanceType: "hamming",
}),
});
const query = Array(32)
.fill(1)
.map(() => Math.floor(Math.random() * 255));
const results = await tbl.query().nearestTo(query).limit(10).toArray();
// --8<-- [end:binary_search]
expect(results.length).toBeLessThanOrEqual(10);
}
// ---- Enterprise-style prefilter / postfilter / batch search ----
{
const dimensions = 768;
const rows = Array.from({ length: 50 }, (_, i) => ({
vector: Array.from({ length: dimensions }, () => Math.random() * 2 - 1),
text: `story ${i}`,
keywords: `kw${i}`,
label: i % 4,
}));
await db.createTable("lancedb-enterprise-quickstart", rows, {
mode: "overwrite",
});
{
// --8<-- [start:vector_search_prefilter]
// Generate a sample 768-dimension embedding vector (typical for BERT-based models)
// In real applications, you would get this from an embedding model
const dimensions = 768;
const queryEmbed = Array.from(
{ length: dimensions },
() => Math.random() * 2 - 1,
);
// Open table and perform search
const tableName = "lancedb-enterprise-quickstart";
const table = await db.openTable(tableName);
// Vector search with filters (pre-filtering is the default)
const vectorResults = await table
.search(queryEmbed)
.where("label > 2")
.select(["text", "keywords", "label"])
.limit(5)
.toArray();
console.log("Search results (with pre-filtering):");
console.log(vectorResults);
// --8<-- [end:vector_search_prefilter]
// --8<-- [start:vector_search_postfilter]
const vectorResultsWithPostFilter = await (
table.search(queryEmbed) as lancedb.VectorQuery
)
.where("label > 2")
.postfilter()
.select(["text", "keywords", "label"])
.limit(5)
.toArray();
console.log("Vector search results with post-filter:");
console.log(vectorResultsWithPostFilter);
// --8<-- [end:vector_search_postfilter]
// --8<-- [start:batch_search]
// Batch query
console.log("Performing batch vector search...");
const batchSize = 5;
const queryVectors = Array.from({ length: batchSize }, () =>
Array.from({ length: dimensions }, () => Math.random() * 2 - 1),
);
let batchQuery = table.search(queryVectors[0]) as lancedb.VectorQuery;
for (let i = 1; i < batchSize; i++) {
batchQuery = batchQuery.addQueryVector(queryVectors[i]);
}
const batchResults = await batchQuery
.select(["text", "keywords", "label"])
.limit(5)
.toArray();
console.log("Batch vector search results:");
console.log(batchResults);
// --8<-- [end:batch_search]
expect(batchResults.length).toBeGreaterThan(0);
}
}
});
});
docs/web-tests/ts/sentence-transformers.test.ts
+96
View File
@@ -0,0 +1,96 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { expect, test } from "@jest/globals";
import { withTempDirectory } from "./util.ts";
// --8<-- [start:quickstart_imports]
import * as lancedb from "@lancedb/lancedb";
import "@lancedb/lancedb/embedding/transformers";
import { Utf8 } from "apache-arrow";
// --8<-- [end:quickstart_imports]
test("full text search", async () => {
await withTempDirectory(async (databaseDir) => {
const db = await lancedb.connect(databaseDir);
const func = (await lancedb.embedding
.getRegistry()
.get("huggingface")
?.create()) as lancedb.embedding.EmbeddingFunction;
const facts = [
"Albert Einstein was a theoretical physicist.",
"The capital of France is Paris.",
"The Great Wall of China is one of the Seven Wonders of the World.",
"Python is a popular programming language.",
"Mount Everest is the highest mountain in the world.",
"Leonardo da Vinci painted the Mona Lisa.",
"Shakespeare wrote Hamlet.",
"The human body has 206 bones.",
"The speed of light is approximately 299,792 kilometers per second.",
"Water boils at 100 degrees Celsius.",
"The Earth orbits the Sun.",
"The Pyramids of Giza are located in Egypt.",
"Coffee is one of the most popular beverages in the world.",
"Tokyo is the capital city of Japan.",
"Photosynthesis is the process by which plants make their food.",
"The Pacific Ocean is the largest ocean on Earth.",
"Mozart was a prolific composer of classical music.",
"The Internet is a global network of computers.",
"Basketball is a sport played with a ball and a hoop.",
"The first computer virus was created in 1983.",
"Artificial neural networks are inspired by the human brain.",
"Deep learning is a subset of machine learning.",
"IBM's Watson won Jeopardy! in 2011.",
"The first computer programmer was Ada Lovelace.",
"The first chatbot was ELIZA, created in the 1960s.",
].map((text) => ({ text }));
const factsSchema = lancedb.embedding.LanceSchema({
text: func.sourceField(new Utf8()),
vector: func.vectorField(),
});
const tbl = await db.createTable("facts", facts, {
mode: "overwrite",
schema: factsSchema,
});
const query = "How many bones are in the human body?";
const actual = await tbl.search(query).limit(1).toArray();
expect(actual[0].text).toBe("The human body has 206 bones.");
});
}, 100_000);
test.skip("embedding quickstart snippets", async () => {
// --8<-- [start:quickstart_connect]
const db = await lancedb.connect("data/sample-lancedb");
// --8<-- [end:quickstart_connect]
// --8<-- [start:quickstart_init_model]
const model = (await lancedb.embedding
.getRegistry()
.get("huggingface")
?.create()) as lancedb.embedding.EmbeddingFunction;
// --8<-- [end:quickstart_init_model]
// --8<-- [start:quickstart_schema]
const wordsSchema = lancedb.embedding.LanceSchema({
text: model.sourceField(new Utf8()),
vector: model.vectorField(),
});
// --8<-- [end:quickstart_schema]
// --8<-- [start:quickstart_create_table]
const table = await db.createEmptyTable("words", wordsSchema, {
mode: "overwrite",
});
await table.add([{ text: "hello world" }, { text: "goodbye world" }]);
// --8<-- [end:quickstart_create_table]
// --8<-- [start:quickstart_query]
const query = "greetings";
const actual = (await table.search(query).limit(1).toArray())[0];
console.log(actual.text);
// --8<-- [end:quickstart_query]
});
docs/web-tests/ts/storage.test.ts
+187
View File
@@ -0,0 +1,187 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { expect, test } from "@jest/globals";
import * as lancedb from "@lancedb/lancedb";
// --8<-- [start:storage_connect_s3]
async function storageConnectS3() {
const db = await lancedb.connect("s3://bucket/path");
return db;
}
// --8<-- [end:storage_connect_s3]
// --8<-- [start:storage_connect_gcs]
async function storageConnectGcs() {
const db = await lancedb.connect("gs://bucket/path");
return db;
}
// --8<-- [end:storage_connect_gcs]
// --8<-- [start:storage_connect_azure]
async function storageConnectAzure() {
const db = await lancedb.connect("az://bucket/path");
return db;
}
// --8<-- [end:storage_connect_azure]
// --8<-- [start:storage_connect_timeout]
async function storageConnectTimeout() {
const db = await lancedb.connect("s3://bucket/path", {
storageOptions: { timeout: "60s" },
});
return db;
}
// --8<-- [end:storage_connect_timeout]
// --8<-- [start:storage_table_timeout]
async function storageTableTimeout() {
const db = await lancedb.connect("s3://bucket/path");
const table = await db.createTable(
"table",
[{ a: 1, b: 2 }],
{ storageOptions: { timeout: "60s" } },
);
return table;
}
// --8<-- [end:storage_table_timeout]
// --8<-- [start:storage_s3_ddb]
async function storageS3Ddb() {
const db = await lancedb.connect(
"s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
);
return db;
}
// --8<-- [end:storage_s3_ddb]
// --8<-- [start:storage_s3_ddb_local]
async function storageS3DdbLocal() {
const db = await lancedb.connect(
"s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
{
storageOptions: {
endpoint: "http://localhost:4566",
dynamodbEndpoint: "http://localhost:4566",
allowHttp: "true",
},
},
);
return db;
}
// --8<-- [end:storage_s3_ddb_local]
// --8<-- [start:storage_s3_sse_kms]
async function storageS3SseKms() {
const db = await lancedb.connect("s3://bucket/path", {
storageOptions: {
awsServerSideEncryption: "aws:kms",
awsSseKmsKeyId: "<your-kms-key-id-or-arn>",
},
});
return db;
}
// --8<-- [end:storage_s3_sse_kms]
// --8<-- [start:storage_azure_sas]
async function storageAzureSas() {
const db = await lancedb.connect(
"az://my-container/my-database",
{
storageOptions: {
azureStorageAccountName: "some-account",
azureStorageSasToken: "<sas-token>",
},
},
);
return db;
}
// --8<-- [end:storage_azure_sas]
// --8<-- [start:storage_s3_minio]
async function storageS3Minio() {
const db = await lancedb.connect("s3://bucket/path", {
storageOptions: {
region: "us-east-1",
endpoint: "http://minio:9000",
},
});
return db;
}
// --8<-- [end:storage_s3_minio]
// --8<-- [start:storage_s3_express]
async function storageS3Express() {
const db = await lancedb.connect(
"s3://my-bucket--use1-az4--x-s3/path",
{
storageOptions: {
region: "us-east-1",
s3Express: "true",
},
},
);
return db;
}
// --8<-- [end:storage_s3_express]
// --8<-- [start:storage_gcs_service_account]
async function storageGcsServiceAccount() {
const db = await lancedb.connect(
"gs://my-bucket/my-database",
{
storageOptions: {
serviceAccount: "path/to/service-account.json",
},
},
);
return db;
}
// --8<-- [end:storage_gcs_service_account]
// --8<-- [start:storage_azure_account]
async function storageAzureAccount() {
const db = await lancedb.connect(
"az://my-container/my-database",
{
storageOptions: {
accountName: "some-account",
accountKey: "some-key",
},
},
);
return db;
}
// --8<-- [end:storage_azure_account]
// --8<-- [start:storage_tigris_connect]
async function storageTigrisConnect() {
const db = await lancedb.connect(
"s3://your-bucket/path",
{
storageOptions: {
endpoint: "https://t3.storage.dev",
region: "auto",
},
},
);
return db;
}
// --8<-- [end:storage_tigris_connect]
test("storage ts snippets compile", async () => {
expect(storageConnectS3).toBeDefined();
expect(storageConnectGcs).toBeDefined();
expect(storageConnectAzure).toBeDefined();
expect(storageConnectTimeout).toBeDefined();
expect(storageTableTimeout).toBeDefined();
expect(storageS3Ddb).toBeDefined();
expect(storageS3DdbLocal).toBeDefined();
expect(storageS3Minio).toBeDefined();
expect(storageS3Express).toBeDefined();
expect(storageS3SseKms).toBeDefined();
expect(storageGcsServiceAccount).toBeDefined();
expect(storageAzureAccount).toBeDefined();
expect(storageAzureSas).toBeDefined();
expect(storageTigrisConnect).toBeDefined();
});
docs/web-tests/ts/tables.test.ts
+1061
View File
File diff suppressed because it is too large Load Diff
docs/web-tests/ts/tsconfig.json
+17
View File
@@ -0,0 +1,17 @@
{
"include": ["*.test.ts"],
"compilerOptions": {
"target": "es2022",
"module": "NodeNext",
"declaration": true,
"outDir": "./dist",
"strict": true,
"allowJs": true,
"resolveJsonModule": true,
"emitDecoratorMetadata": true,
"experimentalDecorators": true,
"moduleResolution": "NodeNext",
"allowImportingTsExtensions": true,
"emitDeclarationOnly": true
}
}
docs/web-tests/ts/util.ts
+16
View File
@@ -0,0 +1,16 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import * as fs from "node:fs";
import { tmpdir } from "node:os";
import * as path from "node:path";
export async function withTempDirectory(
fn: (tempDir: string) => Promise<void>,
) {
const tmpDirPath = fs.mkdtempSync(path.join(tmpdir(), "temp-dir-"));
try {
await fn(tmpDirPath);
} finally {
fs.rmSync(tmpDirPath, { recursive: true });
}
}
docs/web/agent-branch-experiments.mdx
+100
View File
@@ -0,0 +1,100 @@
---
title: "Run experiments on branches"
sidebarTitle: "Branch Experiments"
description: "Use LanceDB branches to isolate agent-driven experiments from main, evaluate them on a fixed test set, and promote only the winner."
icon: "flask"
keywords: ["AI agents", "branches", "experiments", "embeddings", "evaluation"]
---
LanceDB supports [branching](/tables/branching) to isolate experiments from the main table when working with agents.
Branches are useful when you instruct an agent to try several approaches without
affecting the production data on the `main` branch.
Each experiment gets its own writable table history.
For example, you can generate and compare embeddings from two text embedding
models on the given table:
| Branch | Model | Added column |
| --- | --- | --- |
| `embed-minilm` | `sentence-transformers/all-MiniLM-L6-v2` | `vector_minilm` |
| `embed-nomic` | `nomic-embed-text` from Ollama | `vector_nomic` |
The branches share a logical table, but each has its own schema and history.
The original table handle still points to `main`. LanceDB does not have a
process-wide current branch.
Install the model packages, then download the `nomic-embed-text` model from Ollama
(or any other model you prefer):
```bash
uv add sentence-transformers ollama
ollama pull nomic-embed-text
```
It's always recommended to measure the results of an experiment on a fixed evaluation set.
Here's an example of how you could ask the agent to run both experiments and compare the results:
```text Agent prompt
Use the lancedb plugin and the LanceDB branching documentation to compare two
embedding experiments on the `camelot_multimodal` table.
1. Fork `embed-minilm` from `main`. Build normalized embeddings from
`"{role}. {description}"` with
`sentence-transformers/all-MiniLM-L6-v2` and add them as `vector_minilm`.
2. Fork `embed-nomic` from `main`. Embed the identical text with Ollama
`nomic-embed-text` and add the results as `vector_nomic`.
3. Process rows in batches and write only through each branch-scoped table
handle. Verify that neither new column appears on `main`.
4. Evaluate both branches with the queries "wise magical advisor",
"treacherous rebel", and "virtuous Grail knight". Report the top three names,
latency, and whether the expected character ranks first.
5. Do not modify `main` and do not delete either branch. Recommend a winner
based on the results.
```
<Note>
The vector dimensions may differ because the columns live on separate branches.
Use a different column name for each model. Within an experiment, use the same
model for the stored text and the search queries.
</Note>
## Apply the winning experiment to `main` {#apply-the-winning-experiment-to-main}
The branch experiments leave you with the results side by side, and they
deliberately never touch `main`. Once you've picked a winner, apply it to
`main` yourself by rerunning that experiment's validated transformation
directly against the `main` table. Rerunning the reviewed operation is the
reliable path on both OSS and Enterprise: it replays exactly the transformation
you validated on the branch, and it works the same regardless of how the branch
evolved.
Suppose `nomic-embed-text` wins. Rerun the same Nomic embedding transformation
you validated on `embed-nomic` against `main` in batches, then call
`table.optimize()` on OSS. Verify a bounded vector search on `main`, and keep
the losing branch around until you're confident in the result.
See [Branches](/tables/branching) for more on how branches, versions, and tags
relate.
## More experiments you can run {#more-experiments-you-can-run}
Swapping embedding models on a branch while working with agents is only one example
of what you can do with the LanceDB plugin. The table below shows other experiments
you can run with the plugin.
| Hypothesis | Change on the branch | What to evaluate |
| --- | --- | --- |
| A different embedding model improves retrieval | Add a new vector column | Recall, ranking quality, latency, and cost |
| A new parser or OCR model improves source data | Reprocess files and backfill metadata | Validation failures and a reviewed sample |
| A new search setup works better | Build an index or change hybrid-search and reranking settings | Relevance and latency on a fixed query set |
| A curation rule improves the corpus | Deduplicate, classify, or filter records | False positives, false negatives, and row counts |
| A migration is safe to deploy | Add columns or run a backfill | Schema compatibility and application checks |
For each experiment, ask the agent to state the hypothesis, use a fixed
evaluation set, report the comparison, and, if you want, work on an alternate
branch that's isolated from `main` until you decide you want the new derived
column in `main`.
Once you've chosen a winner, rerun that validated transformation to ingest it into `main`
yourself, on either LanceDB OSS or Enterprise. See our documentation on [branches](/tables/branching)
for more information.
docs/web/api-reference/rest/index.mdx
+84
View File
@@ -0,0 +1,84 @@
---
title: REST API Reference
sidebarTitle: "Overview"
description: "API reference for LanceDB"
---
[Lance REST Namespace](https://lance.org/format/namespace/) spec
is an OpenAPI protocol that enables reading, writing and managing Lance tables by connecting
those metadata services or building a custom metadata server in a standardized way.
LanceDB OSS allows you to interface with Lance tables via the REST Namespace.
LanceDB Enterprise provides an extended REST API with
additional endpoints for managing tables and data.
If you have specific needs or questions about the Enterprise REST API Namespace,
please [contact us](mailto:support@lancedb.com).
## Authentication {#authentication}
<Badge color="red">Enterprise</Badge>
All HTTP requests to LanceDB APIs must contain an <u>x-api-key</u> header that specifies a valid API key and
must be encoded as JSON or Arrow RPC.
To authenticate to the Enterprise REST API, you need the endpoint for your deployment and a valid API key for that deployment.
### Get your Enterprise credentials {#get-your-enterprise-credentials}
1. Obtain the following values from your LanceDB administrator or the LanceDB team that provisioned your Enterprise deployment:
- an API key
- your Enterprise REST endpoint
- your database name, if your deployment uses a private endpoint or `host_override`
2. Export those values in your terminal:
```bash
export LANCEDB_API_KEY="<your-api-key>"
export LANCEDB_URI="https://your-enterprise-endpoint.com"
export LANCEDB_DATABASE="your-database-name"
```
3. If your Enterprise deployment is private, connect through the private network endpoint provided for your deployment. For example, Azure Private Link deployments commonly use a private IP or an internal DNS name as the endpoint.
### Verify authentication {#verify-authentication}
4. Check that you can reach the deployment and list tables:
```bash
curl -X GET "$LANCEDB_URI/v1/management/table" \
-H "Content-Type: application/json" \
-H "x-api-key: $LANCEDB_API_KEY" \
-H "x-lancedb-database: $LANCEDB_DATABASE"
```
If your deployment endpoint already includes the database host name, you can omit the `x-lancedb-database` header.
5. Create a table to confirm write access. Let's call it `words`.
```bash
curl -X POST "$LANCEDB_URI/v1/tables/words" \
-H "Content-Type: application/vnd.apache.arrow.stream" \
-H "x-api-key: $LANCEDB_API_KEY" \
-H "x-lancedb-database: $LANCEDB_DATABASE"
```
6. Check that the table has been created:
```bash
curl -X GET "$LANCEDB_URI/v1/management/table" \
-H "Content-Type: application/json" \
-H "x-api-key: $LANCEDB_API_KEY" \
-H "x-lancedb-database: $LANCEDB_DATABASE"
```
That's it -- you're connected! Now, you can start adding data and querying it.
You can visit the tutorial section to build your own applications with LanceDB.
<Card
title="Tutorials"
icon="book"
href="/tutorials/"
>
Check out our tutorials on building various applications with LanceDB.
</Card>
docs/web/architecture.mdx
+153
View File
@@ -0,0 +1,153 @@
---
title: "Architecture"
sidebarTitle: "Architecture"
description: "How LanceDB is put together, and what changes between running it embedded and running it distributed."
icon: cubes
keywords: ["architecture", "embedded", "distributed", "object storage", "compute", "deployment"]
---
LanceDB has one storage layer and two ways to put compute on top of it. Which
one you run decides where queries execute, what runs on your behalf, and what
you have to operate — and almost nothing else. The data is the same
[Lance](/basics/lance) tables either way.
## One storage layer {#storage}
Everything durable lives in object storage: table data, manifests, indexes. No
machine owns a dataset, and no server has to be running for the data to exist.
```
┌──────────────────────────────────────────┐
│ object storage │
│ table data · manifests · indexes │
└──────────────────────────────────────────┘
```
That is the whole persistence story, and it is why the two deployment shapes can
differ so much above it while staying compatible below it. A table written
embedded is opened by a deployment, and the reverse, without conversion.
## Embedded {#embedded}
```
your process
┌────────────────────────────────┐
│ application │
│ LanceDB library │ ← plans, executes, indexes, compacts
└────────────────────────────────┘
│
┌─────────▼─────────┐
│ object storage │
└───────────────────┘
```
`lancedb.connect("./data")` or `connect("s3://bucket/path")` gives you a
**library**, not a client. Your process opens the tables, plans the queries,
runs the scans, builds the indexes, and writes the new versions. There is no
server in the picture at all.
**What follows from that.** Nothing runs unless you call it — no background
compaction, no server-side [jobs](/basics/jobs), no
[SQL endpoint](/basics/sql). Scale is your process's scale: a query gets the
cores and memory your process has. Access control is whatever your object store
enforces, since there is no other gatekeeper.
**What it is good at.** Development, notebooks, single-node pipelines, embedding
LanceDB inside an application, and anything where "no service to operate" is
worth more than elasticity. Comfortably handles millions of vectors on one
node -- disk-based indexes keep it performant without extra infrastructure.
## Distributed {#distributed}
<Badge color="red">Enterprise</Badge>
```
your process
┌──────────────┐
│ application │
│ LanceDB SDK │ ← a thin client
└──────┬───────┘
│ db:// (REST / Flight SQL)
┌──────▼──────────────────────────────────────┐
│ control plane config · identity · policy │
├─────────────────────────────────────────────┤
│ data plane query serving · writes │
│ indexers indexing · compaction │
└──────┬──────────────────────────────────────┘
│
┌──────▼────────────┐
│ object storage │
└───────────────────┘
```
`connect("db://...")` gives you a **client**. Planning and execution move to a
cluster, and the work splits across tiers that scale independently: a control
plane governing configuration, identity and policy; a data plane serving reads
and writes; and indexers building indexes and compacting outside the request
path.
**What follows from that.** Compute and storage are separated, so a query fleet
scales for interactive traffic without also scaling indexing capacity, and heavy
background work does not steal resources from user queries. Because a server
exists, so do the things that need one: [SQL](/sql),
[functions](/management/function), server-side jobs, and
[authorization](/governance/authorization) per object rather than per bucket.
**What it is good at.** Many concurrent queries, continuous ingestion and
indexing, multi-team access with real permissions, and workloads too large for
one machine -- customers regularly operate on billions of rows and tens of
terabytes.
## What changes, precisely {#differences}
| | Embedded | Distributed |
| --- | --- | --- |
| What you connect to | a library in your process | a cluster |
| Where queries plan and execute | your process | the data plane |
| Table handle | `LanceTable` | `RemoteTable` |
| Indexing and compaction | when you call it | dedicated indexers |
| [Jobs](/basics/jobs) | run in-process, no server record | real jobs: id, status, events, cancel |
| [SQL](/sql) | unavailable | Flight SQL endpoint |
| [Functions](/management/function) | unavailable | run on workers |
| Access control | object-store permissions | [per-object grants](/governance/authorization) |
| Scale bound | your process | the cluster |
| What you operate | nothing | a deployment |
Both expose the same table API — create, open, mutate, query — so application
code moves between them by changing the URI. What does not move is code that
depends on a row in the right-hand column.
## Choosing, and changing your mind {#choosing}
Start embedded. It has no operational cost and the API is the same one you will
keep using.
Move when you hit something in the right-hand column — usually concurrent
serving, ingestion that outgrows one machine, or a permissions requirement.
Because the storage layer is identical, moving is a connection-string change
plus a data migration, not a rewrite.
<Note>
The one thing to decide early is **stable row ids**: the SDK can give them to a
table only when it creates one on a local path or an object store, and a
deployment may require them on a
[materialized view](/management/materialized-view) source. See
[Limits](/basics/limits#schema).
</Note>
## Where to go next {#next}
<CardGroup cols={2}>
<Card title="Guarantees" icon="shield-check" href="/basics/guarantees">
What a write promises, and what a reader sees.
</Card>
<Card title="Lance format" icon="layer-group" href="/basics/lance">
The open format underneath both shapes.
</Card>
<Card title="Enterprise architecture" icon="cubes" href="/enterprise/architecture">
The distributed tiers in detail.
</Card>
<Card title="Limits" icon="ruler" href="/basics/limits">
The bounds you can actually hit.
</Card>
</CardGroup>
docs/web/basics/agentic-skills.mdx
+291
View File
@@ -0,0 +1,291 @@
---
title: "Agentic skills"
sidebarTitle: "Agentic Skills"
description: "Give a coding agent a current, maintained reference for the LanceDB APIs."
icon: robot
keywords: ["agents", "plugin", "claude code", "codex", "cursor", "copilot"]
---
import {
PyCamelotSchema,
PyCamelotBatches,
PyCamelotOssIngestion,
} from '/snippets/build_with_ai_agents.mdx';
A coding agent writing LanceDB code from memory writes whatever was in its
training data -- often an older API, sometimes one that never existed. The
LanceDB agent plugin fixes that by supplying the current reference at the point
the agent needs it.
## The plugin {#plugin}
It ships as the `lancedb` plugin in
the [lancedb-agent-plugins](https://github.com/lancedb/lancedb-agent-plugins)
repository, which is also a plugin marketplace. Install it with your agent's
plugin manager:
<CodeGroup>
```text Claude Code
/plugin marketplace add lancedb/lancedb-agent-plugins
/plugin install lancedb@lancedb
```
```bash Codex icon="terminal"
codex plugin marketplace add lancedb/lancedb-agent-plugins
codex plugin add lancedb@lancedb
```
```bash Other agents icon="terminal"
npx plugins add lancedb/lancedb-agent-plugins
```
</CodeGroup>
The [`plugins`](https://www.npmjs.com/package/plugins) installer shown under
**Other agents** is a cross-tool option. It detects which agent CLIs are on your
`PATH` — Cursor, GitHub Copilot CLI, VS Code, Grok Build, and Kimi Code, as well
as Claude Code and Codex — and installs through each one's native plugin system.
The plugin supplements the agent's training data with current LanceDB
instructions. To pick up later revisions, refresh the marketplace:
<CodeGroup>
```text Claude Code
/plugin marketplace update lancedb
```
```bash Codex icon="terminal"
codex plugin marketplace upgrade
```
```bash Other agents icon="terminal"
npx plugins add lancedb/lancedb-agent-plugins
```
</CodeGroup>
<Accordion title="Where the plugin gets installed">
Each plugin manager clones the repository, reads its marketplace manifest to find
the `lancedb` plugin, and hands the plugin to the agent's own plugin store rather
than copying files into your project. Claude Code and Codex keep it under
`~/.claude/plugins` and `~/.codex/plugins` respectively, so the plugin is available
in every project on the machine.
The `npx plugins` installer defaults to the same user-wide scope. Pass
`--scope project` to record the plugin in the current repository instead, so that
everyone working in it gets the same plugin, or `-t <target>` to install for a
single agent rather than every one it detects:
```bash
npx plugins add lancedb/lancedb-agent-plugins --scope project -t cursor
```
Not every agent supports project scope; `npx plugins targets` lists what it
found and what each target supports.
</Accordion>
### Get started with the LanceDB agent plugin {#get-started-with-the-lancedb-agent-plugin}
This tutorial uses the Camelot dataset from the [quickstart](/quickstart), with
a portrait added for each character. Each LanceDB row contains validated
metadata and raw JPEG bytes. Text, images, and any embeddings you add later
remain in the same table.
### 1\. Download the multimodal dataset {#1-download-the-multimodal-dataset}
From a new project directory, download the JSON file and portraits:
```bash
mkdir -p data/img
BASE_URL="https://docs.lancedb.com/static/assets/tutorials/build-with-ai-agents/camelot/data"
curl -fsSL "$BASE_URL/camelot.json" -o data/camelot.json
for image in \
arthur.jpg \
guinevere.jpg \
merlin.jpg \
mordred.jpg \
sir_galahad.jpg \
sir_gawain.jpg \
sir_lancelot.jpg \
sir_percival.jpg
do
curl -fsSL "$BASE_URL/img/$image" -o "data/img/$image"
done
```
Each JSON record has this shape:
```json
{
"id": 2,
"name": "Merlin",
"role": "Wizard and Advisor",
"description": "A powerful wizard and prophet who mentors Arthur.",
"stats": {
"strength": 2,
"courage": 4,
"magic": 5,
"wisdom": 5
},
"img": "data/img/merlin.jpg"
}
```
JSON input may have missing fields, unexpected fields, or values of the wrong
type. The LanceDB plugin tells the agent to validate each record with strict
Pydantic models before writing it.
After the agent writes the pipeline, inspect the schema, batching, and write
path rather than assuming it followed the plugin's guidance correctly.
### 2\. Prompt your agent to build the pipeline {#2-prompt-your-agent-to-build-the-pipeline}
Install the Python packages used by the example:
```bash icon="terminal"
uv init
uv add lancedb pyarrow pydantic
```
If you're using LanceDB Enterprise, ask the agent to ingest into an Enterprise table,
provide the relevant environment variables for connecting to your Enterprise deployment
in a local `.env` file, and point the agent to it.
```text .env
LANCEDB_URI=db://your_project_name
LANCEDB_API_KEY=your_api_key_here
LANCEDB_REGION=us-east-1
LANCEDB_HOST_OVERRIDE=https://your-enterprise-endpoint.com
```
If you're using LanceDB OSS, no connection settings are required, as it runs as an
embedded retrieval library. A simple prompt like this should work:
```text Agent prompt
# If using OSS
Use the lancedb plugin to ingest the dataset in `data/` into a LanceDB
OSS table.
# If using enterprise
Use the lancedb plugin to ingest the dataset in `data/` into a LanceDB
Enterprise table using the connection information in `.env`.
```
Because the plugin is registered with the agent's own plugin system, the agent
should pick it up on its own once you restart the session. If it does not,
simply ask it to use the `lancedb` plugin in the prompt, as shown above.
That should be enough! The agent will create `ingest_multimodal.py`, or similar.
The following sections inspect the script to verify that it follows the plugin's guidance.
#### Data validation {#data-validation}
The plugin encourages the agent to validate each record with Pydantic before writing it.
The agent should ideally define a schema for the table and a nested schema for the
`stats` field.
<CodeBlock filename="ingest_multimodal.py" language="Python" icon="python">
{PyCamelotSchema}
</CodeBlock>
In this case, our agent correctly defined `Character` and `Stats` Pydantic models
and validated the JSON before adding it to the table.
#### Batched ingestion {#batched-ingestion}
Naively calling `table.add()` once per row is slow, and is considered an anti-pattern
in LanceDB. The plugin encourages the agent to collect incoming rows into batches and
write them with a single `table.add()` call. When you use the plugin, the agent should
produce something like this:
<CodeBlock filename="ingest_multimodal.py" language="Python" icon="python">
{PyCamelotBatches}
</CodeBlock>
The script calls `Character.model_validate(...)` before adding a record to the
batch. If validation fails, that batch is never written. The function yields up
to `batch_size` rows at a time, providing an iterable of batches for the ingestion step,
shown next.
#### Table maintenance {#table-maintenance}
For LanceDB OSS, the plugin instructs the agent to call
`table.optimize()` after the ingestion loop. This compacts small fragments, cleans up
old versions according to the retention policy, and incorporates new data into indexes.
<CodeBlock filename="ingest_multimodal.py" language="Python" icon="python">
{PyCamelotOssIngestion}
</CodeBlock>
If you're using LanceDB Enterprise, the plugin mentions that this step is not needed
because LanceDB Enterprise handles maintenance automatically.
<Note>
This example dataset has only eight rows, so the default batch size writes it in
one call. Larger inputs should still avoid single-row write commits.
</Note>
### 3\. Run the OSS pipeline {#3-run-the-oss-pipeline}
```bash
uv run python ingest_multimodal.py
```
The table now contains the validated character records and their JPEG bytes.
Here are the first three rows:
| Image | Character | Role |
| --- | --- | --- |
| <img src="/static/assets/tutorials/build-with-ai-agents/camelot/data/img/arthur.jpg" alt="King Arthur" width="128" height="128" /> | King Arthur | King of Camelot |
| <img src="/static/assets/tutorials/build-with-ai-agents/camelot/data/img/merlin.jpg" alt="Merlin" width="128" height="128" /> | Merlin | Wizard and Advisor |
| <img src="/static/assets/tutorials/build-with-ai-agents/camelot/data/img/guinevere.jpg" alt="Queen Guinevere" width="128" height="128" /> | Queen Guinevere | Queen of Camelot |
Once your pipeline works, you can [run experiments on branches](/agent-branch-experiments)
to try new embedding models, parsers, or search settings without touching `main`.
### Takeaways {#takeaways}
The example in this tutorial was small, but similar ideas apply to other workflows, too.
Give the agent the data source, the constraints it must respect, and the
artifacts it should return.
The plugin supplies LanceDB-specific guidance, but it's the user's responsibility to
ensure the output makes sense for the application.
### Try the plugin with your own dataset {#try-the-plugin-with-your-own-dataset}
The plugin shown in this tutorial should generalize reasonably well to other use cases.
If you find any issues, open [an issue](https://github.com/lancedb/lancedb-agent-plugins/issues)
on GitHub, clearly describing the intended behavior.
You can choose OSS or Enterprise based on how the work will run:
Start with [LanceDB OSS](/quickstart) during the early stages of a project
when an agent is helping you prototype,
explore a dataset, or run small workflows on a subset of the data. The application owns
the storage and lifecycle work, so ask the agent to validate inputs, write in
batches, and it will include table maintenance operations such as `optimize()`
where appropriate.
Choose [LanceDB Enterprise](/enterprise) when the resulting table becomes
shared production infrastructure, and the workload needs distributed capacity,
private deployment, or platform-managed operations. The underlying data format and table
API stay the same, so the pipeline does not need to be redesigned. The agent
instead connects to a remote `db://` table and lets the cluster handle
maintenance and background work.
## What agents are good at here {#workflows}
The walkthrough above is one pattern. The other has its own tutorial.
<Columns cols={2}>
<Card title="Run experiments on branches" icon="flask" href="/agent-branch-experiments">
Isolate agent-driven experiments on a branch, evaluate them on a fixed test
set, and promote only the winner.
</Card>
</Columns>
It is the more interesting pattern. [Branches](/management/table#versioning)
give an agent somewhere to be wrong safely: it can rewrite a schema, recompute a
column, or try three embedding models without touching `main`, and you keep only
what measured better.
docs/web/basics/catalog.mdx
+107
View File
@@ -0,0 +1,107 @@
---
title: "Lance Catalog"
sidebarTitle: "Lance Catalog"
description: "The naming layer: what objects exist, how they nest, and who may reach them."
icon: sitemap
keywords: ["catalog", "namespace", "hierarchy", "governance", "database", "table", "view", "function"]
---
A catalog answers one question: **what exists, and what is it called.** It holds
names, the hierarchy they sit in, and the rules about who may reach them. It
does not store rows, run queries, or build indexes -- that is the
[lakehouse](/basics/lakehouse) on top of it.
Keeping the two apart is what lets a Lance table be reached by something that is
not LanceDB. The catalog is a specification -- the
[Lance Namespace](/basics/lance#standard) -- so any engine that implements it
resolves the same names to the same tables.
## The objects {#objects}
| Object | What it is |
| --- | --- |
| [Database](/management/database) | The root. A connection opens exactly one. |
| [Namespace](/management/namespace) | A named grouping inside a database. Can nest. |
| [Table](/management/table) | Rows and columns, versioned. What most operations act on. |
| [Materialized view](/management/materialized-view) | A table whose contents come from a query, computed and stored. |
| [View](/management/view) | A saved query. Computed when read, not stored. |
| [Function](/management/function) | Registered code, applied to produce column values. |
```
database
└── namespace
├── table
├── materialized view (a table, defined by a query)
├── view (a query, not stored)
└── function
```
A namespace can contain further namespaces, so the hierarchy goes as deep as
your organisation needs. A table is reached by its path through it:
`prod.events.clicks`.
This page is about what the objects *are*. For the operations on them, see
[Management](/management).
## Resolution {#resolution}
A name is resolved by walking the path, and the walk is the point: reaching a
table means traversing the namespaces above it. That is what makes the hierarchy
more than decoration -- it is the unit of both organisation and access.
A flat set of tables needs none of this. Namespaces start earning their keep when
several teams, environments or domains share one catalog and you want
`prod.search.documents` and `staging.search.documents` to be unambiguous.
## Governance {#governance}
<Badge color="red">Enterprise</Badge>
Because the catalog knows the hierarchy, it is the layer that can enforce
access. A grant names **one privilege** on **one object** for **one subject**,
and it inherits down the path: reaching a table requires `USAGE` on the
namespace containing it and on the database containing that.
That is the practical reason to put a boundary in the hierarchy -- a namespace
per team gives you something to grant on. See
[Authorization](/governance/authorization).
Embedded LanceDB has no such layer. Access is whatever the object store enforces
on the bucket, which has no concept of a table.
## Views and materialized views are different objects {#views}
The names are close enough to mislead. Both are catalog objects defined by a
query over other tables; the difference is when that query runs.
- A **materialized view** stores its rows, and a refresh brings them up to
date. Reads are reads of stored data -- fast, and as current as the last
refresh.
- A **view** stores only the query. Every read runs it, so a view is never stale
and never faster than the query it wraps.
Reach for a materialized view when the query is expensive and the data changes
less often than it is read. Reach for a view when correctness matters more than
latency, or when the query is cheap.
## What the catalog does not hold {#elsewhere}
**Indexes** belong to a table rather than standing alone. Creating one is a
performance decision, not a structural one -- see [Indexing](/indexing).
**Jobs** are not stored data at all; they are handles to work in progress. See
[Jobs](/basics/jobs).
**Rows** are in the [Lance](/basics/lance) files the catalog points at. The
catalog knows a table exists and where it lives; it does not know what is in it.
## Next {#next}
<CardGroup cols={2}>
<Card title="Lance Lakehouse" icon="layer-group" href="/basics/lakehouse">
What operates on these objects: search, compute, maintenance.
</Card>
<Card title="Management" icon="folder-tree" href="/management">
Creating, listing and dropping each one.
</Card>
</CardGroup>
docs/web/basics/guarantees.mdx
+134
View File
@@ -0,0 +1,134 @@
---
title: "Guarantees"
sidebarTitle: "Guarantees"
description: "What a write promises when it returns, what a reader sees, and what survives a failure."
icon: shield-check
keywords: ["guarantees", "consistency", "atomicity", "durability", "concurrency", "acid"]
---
What the system promises, and — just as usefully — what it does not. Most
surprises in practice come from assuming one of the guarantees below is
stronger or weaker than it is.
## Writes {#writes}
**A write is atomic.** Every write commits a new table version. Either the
version exists and holds all of the change, or it does not exist and holds none
of it. There is no state in which a reader sees half an `add()`.
**A write returns only after the commit.** When the call returns, the version
is durable in object storage. Nothing is buffered in your process waiting to be
flushed.
**Every write advances the version**, including one that affects zero rows. A
delete whose predicate matches nothing still commits. Version numbers count
commits, not changes.
**Concurrent writes are resolved optimistically.** A transaction records the
change and the version it was based on. At commit time, a conflicting
concurrent operation is detected rather than silently overwritten, and the
commit is retried against the newer version. Operations that do not collide —
appending while another process appends — both succeed.
<Note>
`merge_insert` reports its retry attempts in the result. A high count means
contention on the same keys, not an error.
</Note>
## Reads {#reads}
**A read sees a consistent snapshot.** A query reads one table version
throughout; it never sees a mixture of two.
**Which version, by default, is the one open at the time the table was opened.**
This is the guarantee most often assumed to be stronger than it is:
<Warning>
A long-lived connection **does not observe other processes' writes** until it
reopens the table or its `read_consistency_interval` elapses. The default is
right for a batch job reading a stable snapshot and wrong for a service
expected to see fresh data.
</Warning>
Set `read_consistency_interval` when several processes write the same table and
readers must observe those writes. The cost is a metadata check on that
interval. See [Consistency](/tables/consistency).
**Your own writes are always visible to you.** The interval governs other
writers, not yourself.
## Durability and history {#durability}
**Old versions are retained, not overwritten.** A new version does not destroy
the one before it, which is what makes [time travel](/management/table#versioning) work
and what lets a reader finish a scan that started on an older version.
**Deletion is a mark, not an erase.** A deleted row is recorded in a deletion
file. The data is recoverable for as long as the version that contains it
exists.
<Warning>
Retention is finite. [Compaction](/maintenance/compaction) prunes versions
older than the retention window — **7 days by default** — and a tag or branch
pointing at a pruned version no longer resolves. If you rely on reading old
versions, widen the window before running it, not after.
</Warning>
**Dropping a table ends its history.** Versioning does not survive
`drop_table`; there is no undrop.
## Jobs {#jobs}
**A refresh recomputes only what changed.** It gives values to rows that lack
them and recomputes rows whose inputs changed since they were computed; it
leaves the rest alone. Two consequences, both relied on elsewhere:
- **Retrying is safe.** Re-running a refresh fills what is still empty or stale.
It is not guaranteed to cost only the rows that did not complete: a retry can
recompute values that were already written.
- **A filled value catches up at the next refresh.** If a column derives from
`price` and a row's `price` later changes, the derived value keeps its old
value until the next refresh recomputes it.
**What a failure keeps depends on where the refresh ran.** `refresh_column` in
your process commits the column once, so a failure keeps none of its values. A
refresh job on a deployment commits finished batches as it goes, so some values
can remain.
**Cancellation is a request.** `cancel()` asks a job to stop; it does not
guarantee it stops before doing more work, and cancelling a finished job does
nothing.
**A job id is opaque.** It correlates with server logs and the jobs API.
Parsing it, or storing it to resume the job later, is not supported.
See [Jobs](/basics/jobs).
## Schema {#schema}
**Schema changes are versioned like data changes.** Adding, renaming, retyping
or dropping a column commits a version, so a schema change can be travelled back
through.
**A computed column's inputs are protected.** While a declaration references a
column, that column cannot be renamed, retyped, or dropped — the declaration is
part of the schema and will not be invalidated out from under itself.
**Most schema changes do not rewrite data.** Adding a derived column writes only
the new column. Type changes are the exception and can be substantial.
## What is not guaranteed {#not-guaranteed}
Worth stating plainly, because each is occasionally assumed:
| | |
| --- | --- |
| Multi-table transactions | A commit covers one table. There is no cross-table atomic write. |
| Read-your-writes across processes | Governed by `read_consistency_interval`, not automatic. |
| Unenforced primary keys | `SET UNENFORCED PRIMARY KEY` records intent; it does not enforce uniqueness. |
| Ordering of concurrent appends | Two appends both succeed; their relative row order is not defined. |
| Index freshness | An index does not cover rows added after it was built until [optimized](/maintenance/compaction). |
| Automatic compaction | Nothing runs in the background. `optimize()` runs when you call it. |
The last two are the most common source of "why is this query slow" — see
[Limits](/basics/limits) for the bounds that pair with them.
docs/web/basics/jobs.mdx
+196
View File
@@ -0,0 +1,196 @@
---
title: "Jobs"
sidebarTitle: "Job"
description: "Work that outlives the call that started it: submitting, waiting, inspecting, cancelling."
icon: hourglass-half
keywords: ["job", "async", "background", "events", "cancel", "refresh"]
---
Some operations take longer than a request should. Building an index over a
large table, refreshing a computed column, registering a function -- these run
on the server after the call that started them has returned.
Each is submitted the same way and returns the same handle.
<Note>
A [query](/basics/query) also has an `_async` form, but its handle streams rows
rather than waiting for an outcome. Both answer the same four questions --
which one is this, how is it going, give me the result, stop it -- and differ
only in the third.
</Note>
## The `_async` convention {#convention}
An operation that can run in the background has two forms. The plain name blocks
until the work is done. The `_async` / `Async` name submits the work and returns
a **job** immediately.
| Blocking | Returns a job | SQL |
| --- | --- | --- |
| `create_index` | `create_index_async` | `CREATE INDEX` |
| `refresh_column` | `refresh_column_async` | `REFRESH COLUMN` |
| `create_function` | `create_function_async` | `CREATE FUNCTION` |
| `drop_table` | `drop_table_async` | `DROP TABLE` |
<Note>
The blocking form is not a different operation -- it submits the same job and
waits. Reach for the `_async` form when you want to do something else meanwhile,
run several at once, or survive the client going away.
</Note>
## Submitting and waiting {#waiting}
<CodeGroup>
```python Python icon="python"
from lancedb.index import IvfPq
job = table.create_index_async("vector", config=IvfPq())
print(job.id) # the server's id for this operation
print(job.status()) # "running" | "finished" | "failed" | "cancelled"
job.wait() # blocks until terminal; raises if it failed
```
```typescript TypeScript icon="square-js"
const job = await table.createIndexAsync("vector");
console.log(job.id); // the server's id for this operation
console.log(await job.status()); // "running" | "finished" | "failed" | "cancelled"
await job.wait(); // resolves when terminal; rejects if it failed
```
</CodeGroup>
`wait` is the one call that raises on failure. `status` is a point snapshot and
returns the failed state rather than raising, which is what you want in a
polling loop.
In Python `wait` takes an optional timeout and returns the operation's typed
result -- a `RefreshColumnJobResult`, a `FunctionVersion`, or `None` for an
operation that has no result:
```python
from datetime import timedelta
result = table.refresh_column_async("embedding").wait(timeout=timedelta(minutes=30))
```
`TimeoutError` means the timeout elapsed, not that the job stopped. The job is
still running; the handle just gave up watching.
<Note>
The TypeScript `wait()` resolves to `void`. Read the outcome from `result` after
it resolves.
</Note>
## Inspecting a job {#inspecting}
Submitting an operation returns only an id, so the detail fields are empty until
the handle fetches the record. `refresh()` does that; `status()` does it as a
side effect.
<CodeGroup>
```python Python icon="python"
job.refresh()
job.state # last observed state, without another round trip
job.job_type # what kind of job, as the server names it
job.creation_ms # when it was created, epoch milliseconds
job.spec # what it was submitted with
job.result # its terminal result, once it succeeds
job.failure # why it failed, when it did
```
```typescript TypeScript icon="square-js"
await job.refresh();
job.state; // last observed state, without another round trip
job.jobType; // what kind of job, as the server names it
job.creationMs; // when it was created, epoch milliseconds
job.spec; // what it was submitted with
job.result; // its terminal result, once it succeeds
job.failure; // why it failed, when it did
```
</CodeGroup>
Printing the handle renders every field it knows, which is usually faster than
reading them one at a time.
## Progress, from events {#events}
`result` only tells you anything once the job has finished. Events are written
as it runs, and they outlive the workers that produced them -- so a job that is
still going, or one that never finished, can still account for what it did.
<CodeGroup>
```python Python icon="python"
events = job.events(limit=5000, filter="state = 'claim_complete'")
```
```typescript TypeScript icon="square-js"
const events = await job.events({ limit: 5000, filter: "state = 'claim_complete'" });
```
```sql SQL
SHOW ERRORS ON my_table FOR JOB 'job-id' LIMIT 100
```
</CodeGroup>
A distributed job records a `claim` / `claim_complete` pair per unit of work,
each carrying `rows_processed`. Summing `rows_processed` over the completed
claims is how you get progress on a job that has no percentage.
`filter` is a SQL-like expression over `state`, `updated_by`, `emitted_from`,
`emitted_by`, and `claim_entity`.
<Warning>
The server returns 1000 event rows by default and 10,000 at most, and
**truncates without telling you**. A job that emits an event per fragment will
silently lose its tail unless you pass `limit`.
</Warning>
Events are recorded server-side, so this raises for an operation that ran
in-process.
## Finding and cancelling jobs {#managing}
The connection lists and reopens jobs, so a handle lost to a restarted process
is recoverable from its id.
<CodeGroup>
```python Python icon="python"
db.list_jobs() # every job the server knows about
job = db.open_job(job_id) # reattach to one
db.cancel_job(job_id) # ask it to stop
```
```typescript TypeScript icon="square-js"
await db.listJobs();
const job = await db.openJob(jobId);
await db.cancelJob(jobId);
```
```sql SQL
SHOW JOBS
```
</CodeGroup>
Cancellation is a request, not a guarantee, and cancelling a job that already
finished does nothing. `JobNotFoundError` means the id does not name a job the
server still has.
<Warning>
A job id is opaque. It correlates with server logs and the calls above, but
parsing it or storing it to resume the job later is not supported.
</Warning>
## In-process jobs {#in-process}
A local database has no server to run anything, so its operations run in your
process. They still return a handle, so the same code works against both -- but
one with nothing behind it: `id` is `None`, `status()` reports `"finished"`,
`wait()` returns immediately, `cancel()` does nothing, and `events()` raises.
This is the difference to expect when moving between embedded and deployed --
see [SDKs](/basics/sdk).
docs/web/basics/lakehouse.mdx
+102
View File
@@ -0,0 +1,102 @@
---
title: "Lance Lakehouse"
sidebarTitle: "Lance Lakehouse"
description: "What operates on the catalog's objects: storage, search, computation, and maintenance."
icon: layer-group
keywords: ["lakehouse", "multimodal", "search", "compute", "maintenance", "feature engineering"]
---
The [catalog](/basics/catalog) says what exists. The **lakehouse** is everything
that acts on it: storing the rows, searching them, computing new ones, and
keeping the whole thing fast as it grows.
The distinction matters because the two layers have different lifetimes. A
catalog entry is a name and a pointer. A lakehouse operation reads and writes
the [Lance](/basics/lance) files that name points at -- and those operations are
where almost all of the cost, and all of the tuning, lives.
## Multimodal is the point {#multimodal}
A lakehouse table holds images, video, audio, text, embeddings, annotations and
the features you derive from them **in one table**, side by side.
That is the difference from assembling a warehouse plus a vector store plus a
feature store. There is one copy of the data, one set of versions, and no
pipeline keeping three systems agreeing about the same rows. A search can filter
on a label, rank by a vector, and return the blob, without leaving the table.
| | |
| --- | --- |
| **Columns** | vectors, blobs, and columns [computed](/feature-engineering/add-computed-column) from other columns |
| **Versions** | every write commits one, so reads can [travel back](/management/table#versioning) |
| **Indexes** | vector, full-text and scalar, built over those columns |
| **Branches** | isolated lines of change you can [merge or discard](/tables/branching) |
## The four capabilities {#capabilities}
Everything the lakehouse does falls into one of four groups, and the rest of
these docs is organized the same way.
### Ingest {#ingest}
Getting rows in, and changing the ones that are there: append, merge, update,
delete. Every write commits a version rather than mutating in place, which is
what makes the other three safe to run against a moving table. See
[Data Ingestion](/ingestion).
### Search {#search}
Vector, full-text and hybrid retrieval over the same rows, with filters that
push into the scan rather than running after it. The search is a lakehouse
operation, not a separate service holding a second copy of your embeddings. See
[Semantic Search](/search).
### Compute {#compute}
Deriving new columns and tables from what is already stored:
[computed columns](/feature-engineering/add-computed-column),
[functions](/management/function) that run Python where the data is, and
[materialized views](/management/materialized-view) that persist a query's
result.
The reason this belongs in the lakehouse rather than in a pipeline outside it is
the format. Adding a derived column writes only that column -- the images and
videos are never rewritten -- so deriving a feature is an ordinary operation
rather than a migration. See [Feature Engineering](/feature-engineering) and
[Data Curation](/curation).
### Maintenance {#maintenance}
The work that keeps the first three fast: building and repairing
[indexes](/indexing), and [compaction](/maintenance/compaction) to merge small
files, prune old versions and fold new rows into existing indexes.
<Warning>
None of it runs on its own. There is no background compactor -- `optimize()`
runs when you call it, and an index does not cover rows added after it was
built until you do. See [Guarantees](/basics/guarantees#not-guaranteed).
</Warning>
## Where the work happens {#where}
The same lakehouse runs in two shapes, and which one you use decides where these
capabilities execute rather than whether you have them:
- **Embedded** -- your process does all four itself, directly against object
storage. Nothing runs unless you call it.
- **Distributed** -- a cluster does them, with indexing and compaction on
workers separate from the ones serving queries, plus
[SQL](/sql) and server-side [jobs](/basics/jobs).
See [Architecture](/architecture) for the comparison in full.
## Next {#next}
<CardGroup cols={2}>
<Card title="Lance Catalog" icon="sitemap" href="/basics/catalog">
The objects these operations act on.
</Card>
<Card title="Lance format" icon="layer-group" href="/basics/lance">
The open format underneath, and why column evolution is cheap.
</Card>
</CardGroup>

Some files were not shown because too many files have changed in this diff Show More