rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-12 23:02:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: align layered RQ pin with current Lance APIs

Browse Source
This commit is contained in:
Yang Cen
2026-10-10 17:05:27 +08:00
parent 4ddd32199c c2c8269fc8
commit 6c9df56d10
524 changed files with 79226 additions and 11327 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.bumpversion.toml
+1 -1
View File
@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.40.0-beta.5"
current_version = "0.41.0-beta.3"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.
.github/release_notes.json
+5 -1
View File
@@ -6,6 +6,10 @@
"title": "## 🏆 Highlights",
"labels": ["highlight"]
},
{
"title": "## 🚨 Critical Fixes",
"labels": ["critical-fix"]
},
{
"title": "## 🛠 Breaking Changes",
"labels": ["breaking-change"]
@@ -38,4 +42,4 @@
"labels": ["ci"]
}
]
}
}
.github/workflows/license-header-check.yml
+1
View File
@@ -9,6 +9,7 @@ on:
- python/**
- nodejs/**
- java/**
- docs/web-tests/**
- .github/workflows/license-header-check.yml
permissions:
.github/workflows/nodejs.yml
+65
View File
@@ -164,6 +164,71 @@ jobs:
python ci/mock_openai.py &
cd nodejs/examples
node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose
arrow-compat:
# The main jobs resolve `apache-arrow` to the version pinned in
# devDependencies (the newest supported release). This job re-runs the
# suite with the library itself running on each older Arrow release the
# peer range still accepts, which is what a consumer on that version gets.
name: Linux (Arrow ${{ matrix.arrow-version }})
timeout-minutes: 30
strategy:
matrix:
arrow-version: [ "19.0.1", "20.0.0" ]
runs-on: "ubuntu-22.04"
defaults:
run:
shell: bash
working-directory: nodejs
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
lfs: true
- uses: pnpm/action-setup@v6
with:
version: 11.1.1
- uses: actions/setup-node@v6
with:
node-version: 24
cache: 'pnpm'
cache-dependency-path: nodejs/pnpm-lock.yaml
- uses: Swatinem/rust-cache@v2
with:
# Same crate, profile and target as the `linux` job above, so reuse
# its cache: rust-cache keys on the job id by default, and this job
# would otherwise start cold on every run until it had saved its own.
shared-key: linux
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Install dependencies
run: |
sudo apt update
sudo apt install -y protobuf-compiler libssl-dev
- name: Build
run: |
pnpm install --frozen-lockfile
# No `--` separator: pnpm forwards it literally, which would
# make napi-rs treat `--profile ci` as a cargo passthrough arg.
pnpm build:debug --profile ci
- name: Resolve apache-arrow to ${{ matrix.arrow-version }}
run: |
# The older releases are already installed as the `apache-arrow-NN`
# test aliases. With pnpm's hoisted linker each alias is a complete,
# self-contained directory, so swap it in for the library's own
# `apache-arrow` instead of running a second install, which has been
# seen to hang after printing "Done" in CI. A copy, not a symlink: the
# cross-version tests rely on the alias being a separate module
# instance from the one the library loads.
major="$(echo '${{ matrix.arrow-version }}' | cut -d. -f1)"
mv node_modules/apache-arrow node_modules/apache-arrow.pinned
cp -R "node_modules/apache-arrow-${major}" node_modules/apache-arrow
# `require("apache-arrow/package.json")` is blocked by the package's exports map.
node -p "'apache-arrow resolved to ' + JSON.parse(require('fs').readFileSync('node_modules/apache-arrow/package.json')).version"
pnpm tsc
- name: Test
env:
# Newer @smithy/core uses dynamic ESM imports.
NODE_OPTIONS: "--experimental-vm-modules"
run: node_modules/.bin/jest --verbose
macos:
timeout-minutes: 30
# macos-15 ships a newer linker; the older macos-14 linker fails to insert
.github/workflows/pypi-publish.yml
+7 -4
View File
@@ -21,7 +21,7 @@ permissions:
contents: read
# Without this, a force-push to a PR leaves the previous run going -- including
# a ~74 minute Windows job and a billed arm64 wheel build.
# a long fat-LTO Windows job and a billed arm64 wheel build.
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
@@ -123,8 +123,11 @@ jobs:
path: target/wheels/lancedb-*.whl
if-no-files-found: error
windows:
timeout-minutes: 90
runs-on: windows-latest
# The fat-LTO build of the `lancedb` crate does not finish within 90 minutes
# on the 4-core standard runner, so use the 8-core larger runner with extra
# headroom.
timeout-minutes: 150
runs-on: windows-2025-8x-x64
env:
# link.exe is single-threaded and the long pole on Windows builds. Use
# rustc's bundled lld-link instead.
@@ -145,7 +148,7 @@ jobs:
# or the default branch -- so with no run on main there is nothing that
# can populate an entry the release build would be allowed to read. Fixing
# this needs a main/nightly trigger (which would also catch wheel-build
# breakage before a release); the ~74 minutes here is otherwise dominated
# breakage before a release); the build time here is otherwise dominated
# by the fat-LTO link, which no cache avoids.
- uses: ./.github/workflows/build_windows_wheel
with:
.github/workflows/rust.yml
+6 -36
View File
@@ -180,7 +180,8 @@ jobs:
env:
CC: clang-18
CXX: clang++-18
GH_TOKEN: ${{ secrets.SOPHON_READ_TOKEN }}
GH_TOKEN: ${{ github.token }}
SOPHON_READ_TOKEN: ${{ secrets.SOPHON_READ_TOKEN }}
steps:
- uses: actions/checkout@v6
with:
@@ -284,7 +285,7 @@ jobs:
runs-on: ubuntu-24.04
strategy:
matrix:
msrv: ["1.91.0"] # This should match up with rust-version in Cargo.toml
msrv: ["1.94.0"] # This should match up with rust-version in Cargo.toml
env:
# Need up-to-date compilers for kernels
CC: clang-18
@@ -307,40 +308,9 @@ jobs:
# unreadable outside their own branch anyway, since GitHub scopes
# caches to the creating ref.
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Downgrade dependencies
# These packages have newer requirements for MSRV
run: |
cargo update -p aws-sdk-bedrockruntime --precise 1.77.0
cargo update -p aws-sdk-dynamodb --precise 1.68.0
cargo update -p aws-config --precise 1.6.0
cargo update -p aws-sdk-kms --precise 1.63.0
cargo update -p aws-sdk-s3 --precise 1.79.0
cargo update -p aws-sdk-sso --precise 1.62.0
cargo update -p aws-sdk-ssooidc --precise 1.63.0
cargo update -p aws-sdk-sts --precise 1.63.0
# aws-runtime/sigv4/credential-types/types and the aws-smithy-*
# crates bumped their MSRV to 1.91.1 in late 2026; pin to the last
# 1.91.0-compatible versions. The order matters — each downgrade
# only succeeds once everything that still pins it at a higher
# version has itself been downgraded.
cargo update -p aws-runtime --precise 1.5.12
cargo update -p aws-types --precise 1.3.9
cargo update -p aws-sigv4 --precise 1.3.5
cargo update -p aws-credential-types --precise 1.2.8
# aws-smithy-checksums must stay at or above 0.63.13: OpenDAL's S3
# service needs crc-fast ~1.9, and older releases pin it to ~1.3.
cargo update -p aws-smithy-checksums --precise 0.63.13
cargo update -p aws-smithy-runtime --precise 1.9.3
cargo update -p aws-smithy-http --precise 0.62.6
cargo update -p aws-smithy-eventstream --precise 0.60.14
cargo update -p aws-smithy-http-client --precise 1.1.3
cargo update -p aws-smithy-observability --precise 0.1.4
cargo update -p aws-smithy-query --precise 0.60.8
cargo update -p aws-smithy-runtime-api --precise 1.9.3
cargo update -p aws-smithy-async --precise 1.2.7
cargo update -p aws-smithy-types --precise 1.3.6
cargo update -p aws-smithy-xml --precise 0.60.11
cargo update -p home --precise 0.5.9
- name: Downgrade dependencies
# aws-smithy-types 1.6.1+ requires rustc 1.94.1
run: cargo update -p aws-smithy-types --precise 1.5.0
- name: cargo +${{ matrix.msrv }} check
env:
RUSTUP_TOOLCHAIN: ${{ matrix.msrv }}
.typos.toml
+15 -1
View File
@@ -1,5 +1,15 @@
[default]
extend-ignore-re = ["(?Rm)^.*(#|//)\\s*spellchecker:disable-line$"]
extend-ignore-re = [
"(?Rm)^.*(#|//)\\s*spellchecker:disable-line$",
# Conventional import aliases in the docs examples: `import fiftyone as fo`
# and `import torchvision.io as tio`.
"\\bas (fo|tio)\\b",
"\\btio\\.",
# The substring query in the n-gram full-text search example.
'search\("nce"',
# Garbled OCR output quoted verbatim in the VLM fine-tuning tutorial.
"permission nstallation",
]
[default.extend-words]
# Azure Kubernetes Service, mentioned in rust/lancedb/src/remote/oauth.rs.
@@ -12,6 +22,10 @@ mmaped = "mmaped"
# `WriteableBuffer` is the real name of a type from Python's `_typeshed` stubs,
# used in python/python/lancedb/_blob.py.
Writeable = "Writeable"
# Superlinked's self-hosted inference engine (SIE), its `sie-lancedb` package,
# and the `sie` embedding function it registers, in
# docs/web/integrations/embedding/superlinked.mdx.
SIE = "SIE"
[files]
extend-exclude = [
CONTRIBUTING.md
+44 -1
View File
@@ -71,7 +71,50 @@ are wrappers around the Rust library.
* `nodejs`: Typescript package source code
* `node`: **Deprecated** Typescript package source code
* `java`: Java package source code
* `docs`: Documentation source code
* `docs/src`: SDK reference, built with mkdocs
* `docs/web`: the open-source pages of [docs.lancedb.com](https://docs.lancedb.com)
* `docs/web-tests`: the tests the documentation's code examples are extracted from
## Documentation
The open-source pages of [docs.lancedb.com](https://docs.lancedb.com) live in
`docs/web`, so a change to behaviour and the change to the page describing it
belong in the same pull request.
Preview them locally — this serves `docs/web` on its own, with no other checkout:
```bash
npm i -g mint # https://mintlify.com/docs/installation
cd docs/web && mint dev
```
This repository holds a page at every path the site publishes, so the whole
navigation resolves locally. Enterprise pages exist here as open-source pages --
what the capability is, and what the embedded form does instead. A separate
private repository supplies the fuller version of each, which replaces the page
at the same path when the published site is assembled.
Two things to know before editing:
**Code examples are not written into the pages.** They live in real tests under
`docs/web-tests/{py,ts,rs}`, are extracted into `docs/web/snippets/`, and are
imported by the pages. Edit the test, then regenerate from the repository root:
```bash
uv run docs/web-tests/mdx_snippets_gen.py -s docs/web-tests/py -s docs/web-tests/ts -s docs/web-tests/rs -o docs/web/snippets
```
Review the diff before committing: a few modules currently regenerate with
unrelated changes, so commit only the snippets for the examples you changed.
Run the Python, TypeScript and Rust examples in `docs/web-tests/{py,ts,rs}`
locally before submitting the change. These examples are not yet wired into
pull-request CI; that integration remains follow-up work.
**Every heading carries an explicit `{#anchor}`.** Those anchors are how the
Enterprise pages attach their additions to the right section, so they are
assigned once and never regenerated. Leave an existing anchor alone even when
you reword the heading above it; only new headings need a new one.
## Release process
Cargo.lock
Generated
+183 -282
View File
File diff suppressed because it is too large Load Diff
Cargo.toml
+16 -16
View File
@@ -10,23 +10,23 @@ repository = "https://github.com/lancedb/lancedb"
description = "Serverless, low-latency vector database for AI applications"
keywords = ["lancedb", "lance", "database", "vector", "search"]
categories = ["database-implementations"]
rust-version = "1.91.0"
rust-version = "1.94.0"
[workspace.dependencies]
lance = { "version" = "=13.0.0-beta.9", default-features = false, "rev" = "68f188b5f34dc461a9660c43cdbe4858f42a6d84", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=13.0.0-beta.9", "rev" = "68f188b5f34dc461a9660c43cdbe4858f42a6d84", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=13.0.0-beta.9", "rev" = "68f188b5f34dc461a9660c43cdbe4858f42a6d84", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=13.0.0-beta.9", "rev" = "68f188b5f34dc461a9660c43cdbe4858f42a6d84", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=13.0.0-beta.9", default-features = false, "rev" = "68f188b5f34dc461a9660c43cdbe4858f42a6d84", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=13.0.0-beta.9", "rev" = "68f188b5f34dc461a9660c43cdbe4858f42a6d84", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=13.0.0-beta.9", "rev" = "68f188b5f34dc461a9660c43cdbe4858f42a6d84", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=13.0.0-beta.9", "rev" = "68f188b5f34dc461a9660c43cdbe4858f42a6d84", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=13.0.0-beta.9", default-features = false, "rev" = "68f188b5f34dc461a9660c43cdbe4858f42a6d84", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=13.0.0-beta.9", "rev" = "68f188b5f34dc461a9660c43cdbe4858f42a6d84", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=13.0.0-beta.9", "rev" = "68f188b5f34dc461a9660c43cdbe4858f42a6d84", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=13.0.0-beta.9", "rev" = "68f188b5f34dc461a9660c43cdbe4858f42a6d84", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=13.0.0-beta.9", "rev" = "68f188b5f34dc461a9660c43cdbe4858f42a6d84", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=13.0.0-beta.9", "rev" = "68f188b5f34dc461a9660c43cdbe4858f42a6d84", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=14.0.0-beta.10", default-features = false, "rev" = "81bb501f9e959d607e3c3d77eed7a872252f255f", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=14.0.0-beta.10", "rev" = "81bb501f9e959d607e3c3d77eed7a872252f255f", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=14.0.0-beta.10", "rev" = "81bb501f9e959d607e3c3d77eed7a872252f255f", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=14.0.0-beta.10", "rev" = "81bb501f9e959d607e3c3d77eed7a872252f255f", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=14.0.0-beta.10", default-features = false, "rev" = "81bb501f9e959d607e3c3d77eed7a872252f255f", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=14.0.0-beta.10", "rev" = "81bb501f9e959d607e3c3d77eed7a872252f255f", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=14.0.0-beta.10", "rev" = "81bb501f9e959d607e3c3d77eed7a872252f255f", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=14.0.0-beta.10", "rev" = "81bb501f9e959d607e3c3d77eed7a872252f255f", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=14.0.0-beta.10", default-features = false, "rev" = "81bb501f9e959d607e3c3d77eed7a872252f255f", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=14.0.0-beta.10", "rev" = "81bb501f9e959d607e3c3d77eed7a872252f255f", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=14.0.0-beta.10", "rev" = "81bb501f9e959d607e3c3d77eed7a872252f255f", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=14.0.0-beta.10", "rev" = "81bb501f9e959d607e3c3d77eed7a872252f255f", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=14.0.0-beta.10", "rev" = "81bb501f9e959d607e3c3d77eed7a872252f255f", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=14.0.0-beta.10", "rev" = "81bb501f9e959d607e3c3d77eed7a872252f255f", "git" = "https://github.com/lance-format/lance.git" }
lancedb = { path = "rust/lancedb", default-features = false }
ahash = "0.8"
# Note that this one does not include pyarrow
@@ -75,7 +75,7 @@ serde = "1"
serde_json = "1"
tempfile = "3.5.0"
tokio = { version = "1.23", features = ["rt-multi-thread", "sync"] }
tonic = { version = "0.14", features = ["tls-native-roots", "tls-ring"] }
tonic = { version = "0.14", features = ["tls-native-roots", "tls-aws-lc"] }
uuid = { version = "1.7.0", features = ["v4", "v7"] }
chrono = { version = "0.4", default-features = false, features = ["clock"] }
README.md
+2 -4
View File
@@ -1,8 +1,6 @@
<a href="https://cloud.lancedb.com" target="_blank">
<img src="https://github.com/user-attachments/assets/92dad0a2-2a37-4ce1-b783-0d1b4f30a00c" alt="LanceDB Cloud Public Beta" width="100%" style="max-width: 100%;">
</a>
<div align="center">
[![Reverie](docs/src/assets/reverie-banner.png)](https://www.reveriesummit.com/?utm_medium=social&utm_source=github&utm_campaign=reverie-2026)
[![LanceDB](docs/src/assets/hero-header.png)](https://lancedb.com)
[![Website](https://img.shields.io/badge/-Website-100000?style=for-the-badge&labelColor=645cfb&color=645cfb)](https://lancedb.com/)
[![Blog](https://img.shields.io/badge/Blog-100000?style=for-the-badge&labelColor=645cfb&color=645cfb)](https://blog.lancedb.com/)
@@ -13,7 +11,7 @@
<img src="docs/src/assets/lancedb.png" alt="LanceDB" width="50%">
# **The Multimodal AI Lakehouse**
# **The Multimodal Lakehouse for AI**
[**How to Install** ](#how-to-install) ✦ [**Detailed Documentation**](https://docs.lancedb.com) ✦ [**Tutorials and Recipes**](https://github.com/lancedb/vectordb-recipes/tree/main) ✦ [**Contributors**](#contributors)
docs/src/assets/hero-header.png
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 1.7 MiB

After

Width:  |  Height:  |  Size: 1.3 MiB

docs/src/assets/reverie-banner.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 722 KiB

docs/src/java/java.md
+1 -1
View File
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
<dependency>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-core</artifactId>
<version>0.40.0-beta.5</version>
<version>0.41.0-beta.3</version>
</dependency>
```
docs/src/js/classes/AutoQuery.md
+3 -1
View File
@@ -350,7 +350,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
@@ -417,6 +417,8 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
#### Inherited from
`StandardQueryBase.toArrow`
docs/src/js/classes/BlobFile.md
+101 -5
View File
@@ -10,16 +10,52 @@ A lazy handle to blob bytes. Create one with [Table.fetchBlobFiles](Table.md#fet
## Methods
### close()
```ts
close(): Promise<void>
```
Releases the handle. Reads after `close()` fail. Calling it again does
nothing.
#### Returns
`Promise`&lt;`void`&gt;
***
### isClosed()
```ts
isClosed(): boolean
```
Returns true after [BlobFile.close](BlobFile.md#close).
#### Returns
`boolean`
***
### read()
```ts
read(): Promise<Buffer>
read(maxBytes?, options?): Promise<Buffer>
```
Reads from the cursor to the end and advances the cursor.
Reads from the cursor and advances the cursor.
A second call returns an empty buffer. [BlobFile.readRange](BlobFile.md#readrange) does
not move the cursor.
Reads to the end when `maxBytes` is omitted, or at most `maxBytes` bytes
otherwise. Returns an empty buffer at the end of the blob.
[BlobFile.readRange](BlobFile.md#readrange) does not move the cursor.
#### Parameters
* **maxBytes?**: `bigint`
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
@@ -30,7 +66,10 @@ not move the cursor.
### readRange()
```ts
readRange(start, end): Promise<Buffer>
readRange(
start,
end,
options?): Promise<Buffer>
```
Reads the half-open byte range `[start, end)`.
@@ -43,12 +82,55 @@ Fails when `end` is past the blob size. Does not move the cursor.
* **end**: `bigint`
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;`Buffer`&gt;
***
### readRanges()
```ts
readRanges(ranges, options?): Promise<Buffer[]>
```
Reads several half-open byte ranges. Returns one buffer per range, in
the order given.
Fails when any `end` is past the blob size. Does not move the cursor.
#### Parameters
* **ranges**: [`BlobRange`](../type-aliases/BlobRange.md)[]
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;`Buffer`[]&gt;
***
### seek()
```ts
seek(position): Promise<void>
```
Moves the cursor to `position`, in bytes from the start of the blob.
#### Parameters
* **position**: `bigint`
#### Returns
`Promise`&lt;`void`&gt;
***
### size()
```ts
@@ -60,3 +142,17 @@ Returns the blob size in bytes.
#### Returns
`bigint`
***
### tell()
```ts
tell(): Promise<bigint>
```
Returns the cursor position in bytes.
#### Returns
`Promise`&lt;`bigint`&gt;
docs/src/js/classes/Catalog.md
+15 -4
View File
@@ -89,19 +89,30 @@ Drop an empty database. The server rejects nonempty databases.
### listDatabases()
```ts
listDatabases(options): Promise<ListDatabasesResponse>
listDatabases(options): DatabaseNames
```
List one page of databases; pass pageToken from a response for the next page.
Iterate lazily over all database names using `for await...of`.
pageToken resumes from a saved token; omitted starts at the beginning.
pageLimit limits each REST response, not the total; omitted uses the server default.
Request errors are raised during iteration and terminate the iterator.
#### Parameters
* **options** = `{}`
* **options.limit?**: `number`
* **options.pageLimit?**: `number`
* **options.pageToken?**: `string`
#### Returns
`Promise`&lt;[`ListDatabasesResponse`](../interfaces/ListDatabasesResponse.md)&gt;
[`DatabaseNames`](DatabaseNames.md)
#### Example
```ts
for await (const name of catalog.listDatabases({ pageLimit: 20 })) {
console.log(name);
}
```
docs/src/js/classes/Connection.md
+186 -10
View File
@@ -183,10 +183,6 @@ Define a materialized view named `name` over the table `source`.
The view is populated before creation returns. Set `withNoData` to create
only its definition and empty backing table. The view is a normal table:
it can be queried, indexed and searched, and it appears in `tableNames`.
The source table must have stable row ids (create it with
the `newTableEnableStableRowIds` storage option); they keep the view's
provenance valid across source compactions and cannot be enabled after
a table exists.
#### Parameters
@@ -319,6 +315,38 @@ Creates a new Table and initialize it with new data.
***
### createView()
```ts
abstract createView(
name,
query,
namespacePath?): Promise<ViewDescription>
```
Create a view: a named query the database plans on every read.
The query is planned once, at creation, so one that cannot be planned is
rejected now rather than at the first read. A view holds no rows, and its
readers see its sources as they are at read time.
There is no replace: a name already taken is an error, and changing a
view is a drop followed by a create.
#### Parameters
* **name**: `string`
* **query**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;[`ViewDescription`](../interfaces/ViewDescription.md)&gt;
***
### describeNamespace()
```ts
@@ -342,6 +370,27 @@ The namespace's properties
***
### describeView()
```ts
abstract describeView(name, namespacePath?): Promise<ViewDescription>
```
What this database records about the view named `name`: its defining
query and the schema that query resolved to.
#### Parameters
* **name**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;[`ViewDescription`](../interfaces/ViewDescription.md)&gt;
***
### display()
```ts
@@ -498,6 +547,53 @@ on the returned job to know when cleanup has finished.
***
### dropView()
```ts
abstract dropView(name, namespacePath?): Promise<void>
```
Drop the view named `name` and wait for its definition to be deleted.
The tables it reads are untouched: a view holds no rows of its own. Use
[dropViewAsync](Connection.md#dropviewasync) to retain the cleanup job instead of waiting on it.
#### Parameters
* **name**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;`void`&gt;
***
### dropViewAsync()
```ts
abstract dropViewAsync(name, namespacePath?): Promise<Job>
```
Start dropping the view named `name` and return the job deleting its
definition, without waiting for completion.
The name is free before this resolves. When nothing was bound to it, the
returned job is already finished and has no id.
#### Parameters
* **name**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;[`Job`](Job.md)&gt;
***
### isOpen()
```ts
@@ -515,30 +611,42 @@ Return true if the connection has not been closed
### listJobs()
```ts
abstract listJobs(): Promise<JobInfo[]>
abstract listJobs(options?): Listing<JobInfo>
```
List server-side jobs across the database's tables.
Iterate lazily over server-side jobs across the database's tables.
Use `for await...of`. Options control page size and the starting token.
See [Listing](Listing.md) for pagination state and error behavior.
#### Parameters
* **options?**: [`ListingOptions`](../interfaces/ListingOptions.md)
#### Returns
`Promise`&lt;[`JobInfo`](../interfaces/JobInfo.md)[]&gt;
[`Listing`](Listing.md)&lt;[`JobInfo`](../interfaces/JobInfo.md)&gt;
***
### listMaterializedViews()
```ts
abstract listMaterializedViews(): Promise<string[]>
abstract listMaterializedViews(options?): Listing<string>
```
The names of the materialized views in this database.
Found by reading every table's schema, so this costs an open per table.
Iterate lazily with `for await...of`. Local listings inspect table schemas
one page at a time. Options control page size and the starting token.
See [Listing](Listing.md) for pagination state and error behavior.
#### Parameters
* **options?**: [`ListingOptions`](../interfaces/ListingOptions.md)
#### Returns
`Promise`&lt;`string`[]&gt;
[`Listing`](Listing.md)&lt;`string`&gt;
***
@@ -636,6 +744,30 @@ A page of table names and an
***
### listViews()
```ts
abstract listViews(namespacePath?, options?): Listing<string>
```
The names of the views in one namespace.
Names only; a definition comes from [describeView](Connection.md#describeview).
Iterate lazily with `for await...of`. See [Listing](Listing.md) for pagination
state and error behavior; options control page size and the starting token.
#### Parameters
* **namespacePath?**: `string`[]
* **options?**: [`ListingOptions`](../interfaces/ListingOptions.md)
#### Returns
[`Listing`](Listing.md)&lt;`string`&gt;
***
### openJob()
```ts
@@ -703,6 +835,28 @@ abstract openTable(
***
### pauseJob()
```ts
abstract pauseJob(jobId): Promise<string>
```
Pause a server-side job by id.
The job's workers drain and it stays parked until resumed. Resolves to
"pausing", "already_paused", or "committing" -- a job finalizing its
results cannot be parked; retry shortly.
#### Parameters
* **jobId**: `string`
#### Returns
`Promise`&lt;`string`&gt;
***
### renameTable()
```ts
@@ -736,6 +890,28 @@ a "not supported" error.
***
### resumeJob()
```ts
abstract resumeJob(jobId): Promise<string>
```
Resume a paused server-side job by id.
Its workers pick their work back up from checkpoints. Resolves to
"resumed", "still_pausing" -- the pause's worker drain is not confirmed
yet; retry shortly -- or "not_paused".
#### Parameters
* **jobId**: `string`
#### Returns
`Promise`&lt;`string`&gt;
***
### ~~tableNames()~~
#### tableNames(options)
docs/src/js/classes/DatabaseNames.md
+79
View File
@@ -0,0 +1,79 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / DatabaseNames
# Class: DatabaseNames
A lazy async iterator of database names with pagination state.
Returned by [Catalog.listDatabases](Catalog.md#listdatabases).
## Implements
- `AsyncIterableIterator`&lt;`string`&gt;
## Methods
### \[asyncIterator\]()
```ts
asyncIterator: AsyncIterableIterator<string>
```
#### Returns
`AsyncIterableIterator`&lt;`string`&gt;
#### Implementation of
`AsyncIterableIterator.[asyncIterator]`
***
### next()
```ts
next(): Promise<IteratorResult<string, any>>
```
Fetch the next name, requesting another page only when needed.
#### Returns
`Promise`&lt;`IteratorResult`&lt;`string`, `any`&gt;&gt;
#### Implementation of
`AsyncIterableIterator.next`
***
### numPageResults()
```ts
numPageResults(): number
```
Number of names available without another REST request.
#### Returns
`number`
***
### pageToken()
```ts
pageToken(): undefined | string
```
Token for the next REST request. Initially this is the supplied starting token.
Returns undefined after the final page is fetched, even if names remain cached.
Drain the cache before saving a token to avoid skipping names when resuming.
A failed request terminates iteration and retains its token for a new iterator.
#### Returns
`undefined` \| `string`
docs/src/js/classes/Listing.md
+91
View File
@@ -0,0 +1,91 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / Listing
# Class: Listing&lt;T&gt;
A lazy async iterator over database resources with pagination state.
Requests occur during iteration; request errors terminate the iterator.
## Example
```ts
for await (const name of connection.listViews([], { pageLimit: 20 })) {
console.log(name);
}
```
## Type Parameters
• **T**
## Implements
- `AsyncIterableIterator`&lt;`T`&gt;
## Methods
### \[asyncIterator\]()
```ts
asyncIterator: AsyncIterableIterator<T>
```
#### Returns
`AsyncIterableIterator`&lt;`T`&gt;
#### Implementation of
`AsyncIterableIterator.[asyncIterator]`
***
### next()
```ts
next(): Promise<IteratorResult<T, any>>
```
Fetch the next item, requesting another page only when needed.
#### Returns
`Promise`&lt;`IteratorResult`&lt;`T`, `any`&gt;&gt;
#### Implementation of
`AsyncIterableIterator.next`
***
### numPageResults()
```ts
numPageResults(): number
```
Number of cached items available without another request.
#### Returns
`number`
***
### pageToken()
```ts
pageToken(): undefined | string
```
Token for the next request. Initially this is the supplied starting token.
Returns undefined after fetching the final page, even with cached items.
Drain the cache before saving a token to avoid skipping items on resumption.
Failed requests terminate iteration and retain their token for a new iterator.
#### Returns
`undefined` \| `string`
docs/src/js/classes/MaterializedView.md
+3 -10
View File
@@ -65,21 +65,14 @@ refresh(options?): Promise<RefreshMaterializedViewResult>
Recompute the view from its source.
The refresh is incremental when the source's changes can be reconciled
into the view -- rows added, changed or removed since the last one --
and otherwise rebuilds. `full` forces a rebuild; `sourceVersion`
refreshes to that source version instead of the latest.
Concurrent refreshes of one view do not duplicate its rows. Two that
plan the same source rows conflict on commit, and the loser throws
rather than writing them a second time.
Every refresh rebuilds the complete view. For local views,
`sourceVersion` refreshes to that source version instead of the latest;
remote SQL refreshes do not support source-version pinning.
#### Parameters
* **options?**
* **options.full?**: `boolean`
* **options.sourceVersion?**: `number`
#### Returns
docs/src/js/classes/Query.md
+3 -1
View File
@@ -424,7 +424,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
@@ -491,6 +491,8 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
#### Inherited from
`StandardQueryBase.toArrow`
docs/src/js/classes/QueryBase.md
+3 -1
View File
@@ -193,7 +193,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
@@ -252,6 +252,8 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
***
### withRowId()
docs/src/js/classes/Table.md
+22 -9
View File
@@ -516,7 +516,10 @@ Drop an index from the table.
### fetchBlobFiles()
```ts
abstract fetchBlobFiles(column, rowIds): Promise<(null | BlobFile)[]>
abstract fetchBlobFiles(
column,
rowIds,
options?): Promise<(null | BlobFile)[]>
```
Opens lazy blob handles for `column` at the given row IDs using the
@@ -531,6 +534,8 @@ See [Table.fetchBlobs](Table.md#fetchblobs) for row-ID validity across versions.
* **rowIds**: readonly (`number` \| `bigint`)[]
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;(`null` \| [`BlobFile`](BlobFile.md))[]&gt;
@@ -540,7 +545,10 @@ See [Table.fetchBlobs](Table.md#fetchblobs) for row-ID validity across versions.
### fetchBlobs()
```ts
abstract fetchBlobs(column, rowIds): Promise<(null | Buffer)[]>
abstract fetchBlobs(
column,
rowIds,
options?): Promise<(null | Buffer)[]>
```
Bytes for `column` at row IDs from [Query.withRowId](Query.md#withrowid).
@@ -548,6 +556,10 @@ Bytes for `column` at row IDs from [Query.withRowId](Query.md#withrowid).
Reads the table's current checkout. IDs from another version can fail after
compaction unless stable row ids are enabled. Results keep input order and
duplicates. Null blobs are `null`. Empty blobs are empty buffers.
Remote servers limit each request to 1024 row IDs and 64 MiB of blob bytes.
The client splits requests automatically and reads an individual larger
blob through the Range route. This method still materializes all bytes in
memory; use [Table.fetchBlobFiles](Table.md#fetchblobfiles) for large values.
#### Parameters
@@ -555,6 +567,8 @@ duplicates. Null blobs are `null`. Empty blobs are empty buffers.
* **rowIds**: readonly (`number` \| `bigint`)[]
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;(`null` \| `Buffer`)[]&gt;
@@ -648,10 +662,9 @@ Read the [LsmWriteSpec](../interfaces/LsmWriteSpec.md) currently installed on th
Resolves to `undefined` when the MemWAL LSM write path is not enabled (no
spec has been set, or it was removed with [Table#unsetLsmWriteSpec](Table.md#unsetlsmwritespec)).
The returned spec mirrors what was passed to
[Table#setLsmWriteSpec](Table.md#setlsmwritespec), except that `maintainedIndexes` always
reports the concrete list resolved when the spec was set — `undefined`
never round-trips.
The spec is the one installed, including its maintained-index selection:
an absent `maintainedIndexes` for every index the table has, an empty
array for none.
#### Returns
@@ -1093,8 +1106,8 @@ All variants require the table to have an unenforced primary key
([Table#setUnenforcedPrimaryKey](Table.md#setunenforcedprimarykey)); bucket sharding additionally
requires it to be the single column being bucketed.
Omitting `maintainedIndexes` maintains every index on the table, resolved
here, failing if one cannot be maintained — name them to install anyway.
Omitting `maintainedIndexes` maintains every index the table has,
including ones created later, and skips a kind the MemWAL cannot maintain.
Naming them pins an exact set, and a still-building index is rejected
rather than quietly omitted.
@@ -1367,7 +1380,7 @@ repeatedly calling this method.
##### Parameters
* **updates**: `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **updates**: `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
the
columns to update
docs/src/js/classes/TakeQuery.md
+3 -1
View File
@@ -200,7 +200,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
@@ -267,6 +267,8 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
#### Inherited from
[`QueryBase`](QueryBase.md).[`toArrow`](QueryBase.md#toarrow)
docs/src/js/classes/VectorQuery.md
+10 -7
View File
@@ -447,6 +447,8 @@ nprobes(nprobes): VectorQuery
Set the number of partitions to search (probe)
The number of probes must be greater than 0.
This argument is only used when the vector column has an IVF PQ index.
If there is no index then this value is ignored.
@@ -458,17 +460,16 @@ exhaustiely searched to find matches. This parameter controls how many
partitions should be searched.
Increasing this value will increase the recall of your query but will
also increase the latency of your query. The default value is 20. This
default is good for many cases but the best value to use will depend on
your data and the recall that you need to achieve.
also increase the latency of your query. If this method is not called,
Lance's adaptive probe defaults are used.
For best results we recommend tuning this parameter with a benchmark against
your actual data to find the smallest possible value that will still give
you the desired recall.
For more fine grained control over behavior when you have a very narrow filter
you can use `minimumNprobes` and `maximumNprobes`. This method sets both
the minimum and maximum to the same value.
The value is retained as `nprobes` through client and server request
construction. Lance sets both probe bounds to this value. Explicit minimum
or maximum settings can override their respective bounds.
#### Parameters
@@ -673,7 +674,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
@@ -740,6 +741,8 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
#### Inherited from
`StandardQueryBase.toArrow`
docs/src/js/functions/connectCatalog.md
+3 -1
View File
@@ -28,5 +28,7 @@ headers; opened database connections inherit authentication and client options.
```ts
const catalog = await connectCatalog("https://my-server.example", { apiKey: "secret" });
const db = await catalog.createDatabase("analytics", { existOk: true });
const page = await catalog.listDatabases({ limit: 20 });
for await (const name of catalog.listDatabases({ pageLimit: 20 })) {
console.log(name);
}
```
docs/src/js/globals.md
+10 -1
View File
@@ -27,9 +27,11 @@
- [Branches](classes/Branches.md)
- [Catalog](classes/Catalog.md)
- [Connection](classes/Connection.md)
- [DatabaseNames](classes/DatabaseNames.md)
- [HeaderProvider](classes/HeaderProvider.md)
- [Index](classes/Index.md)
- [Job](classes/Job.md)
- [Listing](classes/Listing.md)
- [MakeArrowTableOptions](classes/MakeArrowTableOptions.md)
- [MatchQuery](classes/MatchQuery.md)
- [MaterializedView](classes/MaterializedView.md)
@@ -108,11 +110,11 @@
- [JobEventsOptions](interfaces/JobEventsOptions.md)
- [JobFailureInfo](interfaces/JobFailureInfo.md)
- [JobInfo](interfaces/JobInfo.md)
- [ListDatabasesResponse](interfaces/ListDatabasesResponse.md)
- [ListNamespacesOptions](interfaces/ListNamespacesOptions.md)
- [ListNamespacesResponse](interfaces/ListNamespacesResponse.md)
- [ListTablesOptions](interfaces/ListTablesOptions.md)
- [ListTablesResponse](interfaces/ListTablesResponse.md)
- [ListingOptions](interfaces/ListingOptions.md)
- [LsmStats](interfaces/LsmStats.md)
- [LsmWriteSpec](interfaces/LsmWriteSpec.md)
- [MaterializedViewDefinition](interfaces/MaterializedViewDefinition.md)
@@ -149,6 +151,7 @@
- [UpdateOptions](interfaces/UpdateOptions.md)
- [UpdateResult](interfaces/UpdateResult.md)
- [Version](interfaces/Version.md)
- [ViewDescription](interfaces/ViewDescription.md)
- [WriteExecutionOptions](interfaces/WriteExecutionOptions.md)
- [WriteProgress](interfaces/WriteProgress.md)
@@ -156,9 +159,15 @@
- [AnalyzePlanDistributedMetrics](type-aliases/AnalyzePlanDistributedMetrics.md)
- [BaseTokenizer](type-aliases/BaseTokenizer.md)
- [BlobData](type-aliases/BlobData.md)
- [BlobInput](type-aliases/BlobInput.md)
- [BlobOptions](type-aliases/BlobOptions.md)
- [BlobRange](type-aliases/BlobRange.md)
- [BlobReadOptions](type-aliases/BlobReadOptions.md)
- [BlobUri](type-aliases/BlobUri.md)
- [Data](type-aliases/Data.md)
- [DataLike](type-aliases/DataLike.md)
- [DataTypeLike](type-aliases/DataTypeLike.md)
- [FieldLike](type-aliases/FieldLike.md)
- [IntoSql](type-aliases/IntoSql.md)
- [IntoVector](type-aliases/IntoVector.md)
docs/src/js/interfaces/AddDataOptions.md
+16
View File
@@ -10,6 +10,22 @@ Options for adding data to a table.
## Properties
### allowExternalBlobOutsideBases?
```ts
optional allowExternalBlobOutsideBases: boolean;
```
Whether blob URIs outside registered external bases may be written.
Defaults to `false`. This option is supported only for local/native
tables; remote tables return an error when it is enabled. An enabled write
stores the absolute URI reference without registering a base or copying
the external object into the database. The object must remain accessible
when the blob is read later.
***
### mode
```ts
docs/src/js/interfaces/CherryPickResult.md
+3 -3
View File
@@ -38,9 +38,9 @@ preview: CherryPickPreview;
```ts
status:
| "failed"
| "unknown"
| "ready"
| "failed"
| "notImplemented"
| "cherryPicked";
| "cherryPicked"
| "unknown";
```
docs/src/js/interfaces/DropNamespaceOptions.md
+1 -1
View File
@@ -21,7 +21,7 @@ Refuse to drop if non-empty (restrict) or drop recursively (cascade).
### mode?
```ts
optional mode: "fail" | "skip";
optional mode: "skip" | "fail";
```
Whether to skip if the namespace doesn't exist, or fail.
docs/src/js/interfaces/FtsOptions.md
+2
View File
@@ -37,6 +37,8 @@ The following tokenizers are available:
"raw" - Raw tokenizer. This tokenizer does not split the text into tokens and indexes the entire text as a single token.
"code" - Code tokenizer. This tokenizer keeps snake_case and camelCase identifiers as single tokens.
"icu" - ICU dictionary-based word segmentation.
"icu/split" - ICU segmentation with simple-style delimiter splitting.
docs/src/js/interfaces/ListDatabasesResponse.md
-23
View File
@@ -1,23 +0,0 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / ListDatabasesResponse
# Interface: ListDatabasesResponse
## Properties
### databases
```ts
databases: string[];
```
***
### pageToken?
```ts
optional pageToken: string;
```
docs/src/js/interfaces/ListTablesOptions.md
+1
View File
@@ -15,6 +15,7 @@ optional limit: number;
```
An upper bound on how many tables to return.
Defaults to 100. Zero returns an empty page without a continuation token.
A page may hold fewer than this and still not be the last one, so keep
going while the response carries a page token rather than while pages are
docs/src/js/interfaces/ListingOptions.md
+29
View File
@@ -0,0 +1,29 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / ListingOptions
# Interface: ListingOptions
Pagination controls for lazy resource listings.
## Properties
### pageLimit?
```ts
optional pageLimit: number;
```
Maximum results per request, not a limit on the whole listing.
***
### pageToken?
```ts
optional pageToken: string;
```
Starting continuation token; omitted starts at the beginning.
docs/src/js/interfaces/LsmWriteSpec.md
+2 -3
View File
@@ -34,9 +34,8 @@ Bucket and identity variants: the sharding column.
optional maintainedIndexes: string[];
```
Indexes the MemWAL keeps up to date. Omit to maintain every supported
index, resolved on install — a snapshot, so indexes created later are not
maintained. Pass `[]` for none.
Indexes the MemWAL keeps up to date. Omit for every index the table has,
including ones created later. Pass `[]` for none.
***
docs/src/js/interfaces/MaterializedViewDefinition.md
+2 -2
View File
@@ -8,8 +8,8 @@
The query that defines a materialized view, as stored:
`SELECT columns FROM [ns.]table [, function(args) AS alias | , UNNEST(column) AS alias]
[WHERE predicate] [LIMIT n]`. A Function in `FROM` position yields one row
per element it returns.
[WHERE predicate] [GROUP BY expr, ...] [LIMIT n]`. A Function in `FROM`
position yields one row per element it returns.
## Properties
docs/src/js/interfaces/RefreshMaterializedViewResult.md
+1 -1
View File
@@ -14,7 +14,7 @@
mode: string;
```
How the view was brought up to date: "rebuild", "incremental" or "no_op".
How the view was brought up to date: "rebuild".
***
docs/src/js/interfaces/TableNamesOptions.md
+1 -1
View File
@@ -19,7 +19,7 @@ instead.
optional limit: number;
```
An optional limit to the number of results to return.
The maximum number of names to return. Omitted returns all names; zero returns none.
***
docs/src/js/interfaces/TableStatistics.md
+17
View File
@@ -18,6 +18,23 @@ Statistics on table fragments
***
### numDeletedRows?
```ts
optional numDeletedRows: number;
```
The number of rows marked as deleted across all fragments of the table
These rows are not included in `numRows`, but still occupy space on disk
until the table is compacted, so a large value here indicates that the
table should be optimized. Fragments in which every row was deleted are
dropped outright, so their rows are not counted here.
Absent (`undefined`) when the backend does not report deletion counts.
***
### numIndices
```ts
docs/src/js/interfaces/ViewDescription.md
+78
View File
@@ -0,0 +1,78 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / ViewDescription
# Interface: ViewDescription
What a database records about one view.
A view holds no rows: it stores the statement that defines it and the
schema that statement resolved to, so a reader sees its sources as they
are at read time.
## Properties
### defaultDatabase
```ts
defaultDatabase: string;
```
The database that unqualified table names in [query](ViewDescription.md#query) resolve against.
***
### defaultNamespacePath
```ts
defaultNamespacePath: string[];
```
The namespace path those unqualified names resolve against; empty is the
root namespace.
Recorded with the view because it outlives the session that declared it:
a reader resolving the query against its own default namespace could read
a different table than the view was defined over.
***
### name
```ts
name: string;
```
The view's name within its namespace.
***
### namespacePath
```ts
namespacePath: string[];
```
The namespace holding the view; empty is the root namespace.
***
### query
```ts
query: string;
```
The defining query, as the database stores it.
***
### schema
```ts
schema: Schema<any>;
```
The schema the defining query resolved to when the view was created.
docs/src/js/namespaces/embedding/classes/EmbeddingFunction.md
+2 -2
View File
@@ -52,7 +52,7 @@ new EmbeddingFunction<T, M>(): EmbeddingFunction<T, M>
### computeQueryEmbeddings()
```ts
computeQueryEmbeddings(data): Promise<number[] | Uint8Array | Float32Array | Float64Array>
computeQueryEmbeddings(data): Promise<Float32Array | Float64Array | Uint8Array | number[]>
```
Compute the embeddings for a single query
@@ -63,7 +63,7 @@ Compute the embeddings for a single query
#### Returns
`Promise`&lt;`number`[] \| `Uint8Array` \| `Float32Array` \| `Float64Array`&gt;
`Promise`&lt;`Float32Array` \| `Float64Array` \| `Uint8Array` \| `number`[]&gt;
***
docs/src/js/namespaces/embedding/classes/TextEmbeddingFunction.md
+2 -2
View File
@@ -37,7 +37,7 @@ new TextEmbeddingFunction<M>(): TextEmbeddingFunction<M>
### computeQueryEmbeddings()
```ts
computeQueryEmbeddings(data): Promise<number[] | Uint8Array | Float32Array | Float64Array>
computeQueryEmbeddings(data): Promise<Float32Array | Float64Array | Uint8Array | number[]>
```
Compute the embeddings for a single query
@@ -48,7 +48,7 @@ Compute the embeddings for a single query
#### Returns
`Promise`&lt;`number`[] \| `Uint8Array` \| `Float32Array` \| `Float64Array`&gt;
`Promise`&lt;`Float32Array` \| `Float64Array` \| `Uint8Array` \| `number`[]&gt;
#### Overrides
docs/src/js/type-aliases/BaseTokenizer.md
+1
View File
@@ -12,6 +12,7 @@ type BaseTokenizer:
| "whitespace"
| "raw"
| "ngram"
| "code"
| "icu"
| "icu/split"
| `jieba/${string}`
docs/src/js/type-aliases/BlobData.md
+16
View File
@@ -0,0 +1,16 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobData
# Type Alias: BlobData
```ts
type BlobData: Buffer | Uint8Array | ArrayBuffer | Blob;
```
Bytes accepted for a blob value. `ArrayBuffer` is wrapped without copying.
`Blob` and `File` are read with `arrayBuffer()`, which only the async write
paths ([Connection.createTable](../classes/Connection.md#createtable), [Table.add](../classes/Table.md#add),
[Table.mergeInsert](../classes/Table.md#mergeinsert)) can do.
docs/src/js/type-aliases/BlobInput.md
+36
View File
@@ -0,0 +1,36 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobInput
# Type Alias: BlobInput
```ts
type BlobInput:
| BlobData
| BlobUri
| object
| object
| null
| undefined;
```
A value accepted for a `lance.blob.v2` column (see [blob](../functions/blob.md)).
Either inline bytes, a URI pointing at external bytes, or a struct that sets
exactly one of `data` or `uri`. `null` and `undefined` write a null blob.
## Example
```ts
import { pathToFileURL } from "node:url";
import type { BlobInput } from "@lancedb/lancedb";
const rows: { id: bigint; image: BlobInput }[] = [
{ id: 1n, image: await (await fetch(url)).arrayBuffer() },
{ id: 2n, image: pathToFileURL("/data/cat.png") },
{ id: 3n, image: { data: new Blob(["hello"]) } },
];
await table.add(rows);
```
docs/src/js/type-aliases/BlobRange.md
+27
View File
@@ -0,0 +1,27 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobRange
# Type Alias: BlobRange
```ts
type BlobRange: object;
```
A half-open byte range `[start, end)` for [BlobFile.readRanges](../classes/BlobFile.md#readranges).
## Type declaration
### end
```ts
end: bigint;
```
### start
```ts
start: bigint;
```
docs/src/js/type-aliases/BlobReadOptions.md
+24
View File
@@ -0,0 +1,24 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobReadOptions
# Type Alias: BlobReadOptions
```ts
type BlobReadOptions: object;
```
Options for blob reads.
## Type declaration
### signal?
```ts
optional signal: AbortSignal;
```
Cancels the read. The call rejects with `signal.reason`, and the native
read stops, including in-flight requests to a remote table.
docs/src/js/type-aliases/BlobUri.md
+13
View File
@@ -0,0 +1,13 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobUri
# Type Alias: BlobUri
```ts
type BlobUri: string | URL;
```
A URI accepted for a blob value. A `URL` is stored as its `href`.
docs/src/js/type-aliases/DataTypeLike.md
+18
View File
@@ -0,0 +1,18 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / DataTypeLike
# Type Alias: DataTypeLike
```ts
type DataTypeLike: DataType | object;
```
A `DataType` from any copy or version of apache-arrow.
Arrow 21 brands its classes with `unique symbol` properties, so a type
object from a second copy of the library no longer satisfies the `DataType`
type of this one even though it is structurally identical. Inputs that only
need to be sanitized accept this looser shape instead.
docs/src/python/python.md
+52 -1
View File
@@ -26,6 +26,14 @@ is also an [asynchronous API client](#connections-asynchronous).
::: lancedb.db.DBConnection
Resource listings (`list_secrets`, `list_views`, `list_jobs`, `list_functions`, and
`list_materialized_views`) return lazy iterators. Use `list(db.list_views())` to
collect synchronous results, or `[name async for name in db.list_views()]` with
an asynchronous connection. `page_limit` controls each request, and `page_token`
resumes a listing. Drain the iterator's cached results before saving its token.
::: lancedb.listing.Listing
::: lancedb.Session
## Catalogs (Synchronous)
@@ -37,7 +45,8 @@ are ordinary connections. Dropping a database requires it to be empty.
::: lancedb.catalog.Catalog
::: lancedb.catalog.ListDatabasesResponse
::: lancedb.catalog.DatabaseNames
## Remote SQL
@@ -77,6 +86,15 @@ print(query.describe().status)
for batch in query.reader():
print(batch.num_rows)
# Values bind to $1 / $name placeholders and travel as Arrow, so a float32
# stays a float32 and a vector stays a compact fixed-size list:
import numpy as np
reader = db.execute_query(
"SELECT id FROM docs ORDER BY distance(vector, $vector) LIMIT $k",
parameters={"vector": np.random.rand(768).astype(np.float32), "k": 10},
)
# The async connection exposes the same lifecycle without blocking:
# async_db = await lancedb.connect_async(
# "db://analytics",
@@ -192,12 +210,18 @@ listing a storage directory.
::: lancedb.sql.QueryDescription
::: lancedb.sql.QueryParameters
## Materialized Views (Synchronous)
::: lancedb.materialized_view.MaterializedView
::: lancedb.materialized_view.MaterializedViewDefinition
## Views
::: lancedb.view.ViewDescription
## Expressions
Type-safe expression builder for filters and projections. Use these instead
@@ -375,6 +399,29 @@ still work. Queries return descriptors. Call
## Reranking
`TypeSafeReranker` supports opt-in request batching:
```python
from lancedb.rerankers import TypeSafeReranker
reranker = TypeSafeReranker(batch_size=40, max_concurrency=8)
```
The default `batch_size=1` keeps the query and document in request state and
sends one request per non-null candidate. With `batch_size=40`, 80 non-null
candidates require two requests. `max_concurrency` still limits simultaneous
requests, and the SDK handles retries.
In batched mode, state contains only `{"query": query}`. Each independent
question contains `{"question": instructions, "document": document}` in its
structured instructions, so it sees only its own document and the shared query.
Custom instructions and criteria are kept verbatim: adapt prompts that explicitly
reference request-state fields such as `state.document` before enabling batching.
Null documents retain a zero score without an API call; empty strings are scored.
API failures, mismatched answer IDs, and invalid probabilities raise errors.
Batching changes the payload and can affect model scores; compare quality and
latency on your workload before opting in.
::: lancedb.rerankers
options:
show_root_heading: false
@@ -386,6 +433,8 @@ still work. Queries return descriptors. Call
::: lancedb.catalog.AsyncCatalog
::: lancedb.catalog.AsyncDatabaseNames
Connections represent a connection to a LanceDb database and
can be used to create, list, or open tables.
@@ -393,6 +442,8 @@ can be used to create, list, or open tables.
::: lancedb.db.AsyncConnection
::: lancedb.listing.AsyncListing
## Namespaces (Asynchronous)
::: lancedb.connect_namespace_async
docs/web-tests/camelot.json
+66
View File
@@ -0,0 +1,66 @@
[
{
"id": 1,
"name": "King Arthur",
"role": "King of Camelot",
"description": "The legendary ruler of Camelot, wielder of Excalibur, and leader of the Knights of the Round Table.",
"vector": [0.72, -0.28, 0.60, 0.86],
"stats": { "strength": 2, "courage": 5, "magic": 1, "wisdom": 4 }
},
{
"id": 2,
"name": "Merlin",
"role": "Wizard and Advisor",
"description": "A powerful wizard and prophet who mentors Arthur and shapes the destiny of Camelot through magic and foresight.",
"vector": [0.05, 0.88, 0.62, 0.85],
"stats": { "strength": 2, "courage": 4, "magic": 5, "wisdom": 5 }
},
{
"id": 3,
"name": "Queen Guinevere",
"role": "Queen of Camelot",
"description": "Arthur's queen, admired for her grace and diplomacy, whose romances and loyalties influence Camelot's fate.",
"vector": [0.22, -0.22, 0.42, 0.82],
"stats": { "strength": 1, "courage": 3, "magic": 1, "wisdom": 4 }
},
{
"id": 4,
"name": "Sir Lancelot",
"role": "Knight of the Round Table",
"description": "Arthur's most skilled knight, famed for unmatched combat prowess and his tragic love for Queen Guinevere.",
"vector": [0.86, -0.35, 0.38, 0.55],
"stats": { "strength": 5, "courage": 5, "magic": 1, "wisdom": 3 }
},
{
"id": 5,
"name": "Sir Gawain",
"role": "Knight of the Round Table",
"description": "A noble and honorable knight known for his courtesy and his encounter with the Green Knight.",
"vector": [0.82, -0.32, 0.52, 0.60],
"stats": { "strength": 4, "courage": 5, "magic": 1, "wisdom": 4 }
},
{
"id": 6,
"name": "Sir Galahad",
"role": "Knight of the Round Table",
"description": "The purest and most virtuous knight, chosen to achieve the Holy Grail due to his unwavering spiritual purity.",
"vector": [0.80, -0.20, 0.70, 0.78],
"stats": { "strength": 4, "courage": 5, "magic": 2, "wisdom": 5 }
},
{
"id": 7,
"name": "Sir Percival",
"role": "Knight of the Round Table",
"description": "A loyal and innocent knight whose bravery and sincerity make him one of the key seekers of the Holy Grail.",
"vector": [0.78, -0.36, 0.48, 0.52],
"stats": { "strength": 4, "courage": 4, "magic": 1, "wisdom": 3 }
},
{
"id": 8,
"name": "Mordred",
"role": "Traitor Knight",
"description": "Arthur's treacherous son or nephew who ultimately rebels against him, leading to Camelot's downfall.",
"vector": [0.68, -0.30, -0.65, 0.20],
"stats": { "strength": 4, "courage": 2, "magic": 1, "wisdom": 2 }
}
]
docs/web-tests/mdx_snippets_gen.py
+362
View File
@@ -0,0 +1,362 @@
"""
Generate MDX-ready snippet modules that export raw string constants.
The generator groups snippets by the originating test file name rather than by
language. For a source file like `tests/py/test_basic_usage.py`, the snippets are
written under `docs/snippets/basic.mdx`. Each exported constant is prefixed with
a language identifier (Py|Ts|Rs) followed by the snippet name converted to
TitleCase so that docs authors can selectively assemble `<CodeGroup>` blocks.
"""
from __future__ import annotations
import argparse
import json
import os
import re
from dataclasses import dataclass
from pathlib import Path
from typing import Dict, Iterable, Iterator, List, Mapping, MutableMapping, Tuple
# Supported languages and relevant metadata.
LANG_ORDER = ("py", "ts", "rs")
LANG_PREFIX = {"py": "Py", "ts": "Ts", "rs": "Rs"}
LANG_EXTENSIONS = {
".py": "py",
".ts": "ts",
".tsx": "ts",
".rs": "rs",
}
LANG_MARKERS = {"py": ["#"], "ts": ["//"], "rs": ["//"]}
# Match each export block; we rely on the blank line we emit after each export to
# avoid accidentally truncating on semicolons inside the JSON string literal.
EXPORT_RE = re.compile(r"export const (\w+)\s*=\s*(.+?);\s*\n(?:\s*\n|$)", re.DOTALL)
DEFAULT_SOURCE_DIRS = (
Path("tests") / "py",
Path("tests") / "ts",
Path("tests") / "rs",
)
@dataclass(frozen=True)
class SnippetRecord:
lang: str
snippet_name: str
export_name: str
text: str
source_rel: str
def build_markers(
comment_markers: Iterable[str],
) -> Tuple[re.Pattern[str], re.Pattern[str]]:
escaped = "|".join(re.escape(c) for c in comment_markers)
start_re = re.compile(rf"^\s*(?:{escaped})\s*--8<--\s*\[start:([^\]]+)\]")
end_re = re.compile(rf"^\s*(?:{escaped})\s*--8<--\s*\[end:([^\]]+)\]")
return start_re, end_re
def parse_snippets(
lines: List[str], start_re: re.Pattern[str], end_re: re.Pattern[str]
) -> Dict[str, List[str]]:
snippets: Dict[str, List[str]] = {}
stack: List[Tuple[str, int]] = []
for idx, line in enumerate(lines):
m_start = start_re.match(line)
if m_start:
name = m_start.group(1).strip()
stack.append((name, idx))
continue
m_end = end_re.match(line)
if m_end:
name = m_end.group(1).strip()
if not stack:
raise ValueError(
f"End marker for '{name}' found at line {idx + 1} without matching start"
)
start_name, start_idx = stack.pop()
if start_name != name:
raise ValueError(
f"Mismatched markers: start '{start_name}' at line {start_idx + 1}, "
f"end '{name}' at line {idx + 1}"
)
snippets[name] = lines[start_idx + 1 : idx]
continue
if stack:
open_names = ", ".join(f"{n}@{i + 1}" for n, i in stack)
raise ValueError(f"Unclosed snippet markers for: {open_names}")
return snippets
def dedent(lines: List[str]) -> List[str]:
min_indent = None
for ln in lines:
if ln.strip() == "":
continue
expanded = ln.expandtabs(4)
indent = len(expanded) - len(expanded.lstrip(" "))
if min_indent is None or indent < min_indent:
min_indent = indent
if not lines:
return []
if min_indent in (None, 0):
return [ln.rstrip("\n") for ln in lines]
dedented: List[str] = []
for ln in lines:
expanded = ln.expandtabs(4)
if expanded.strip() == "":
dedented.append(expanded.rstrip("\n"))
else:
dedented.append(expanded[min_indent:].rstrip("\n"))
return dedented
def ensure_dir(path: Path) -> None:
path.mkdir(parents=True, exist_ok=True)
def detect_language(file_path: Path) -> str:
lang = LANG_EXTENSIONS.get(file_path.suffix.lower())
if not lang:
raise ValueError(f"Unsupported file extension for {file_path}")
return lang
def markers_for_lang(lang: str) -> Tuple[re.Pattern[str], re.Pattern[str]]:
markers = LANG_MARKERS.get(lang)
if not markers:
raise ValueError(f"Unsupported language for markers: {lang}")
return build_markers(markers)
def normalize_target_name(file_path: Path) -> str:
stem = file_path.stem
if stem.startswith("test_"):
stem = stem[len("test_") :]
if stem.endswith("_test"):
stem = stem[: -len("_test")]
if stem.endswith(".test"):
stem = stem[: -len(".test")]
if stem.endswith(".spec"):
stem = stem[: -len(".spec")]
if stem == "":
stem = file_path.stem
return stem.replace(" ", "_")
def to_title_case(name: str) -> str:
parts = re.split(r"[^0-9a-zA-Z]+", name)
filtered = [p for p in parts if p]
return "".join(p[:1].upper() + p[1:].lower() for p in filtered) or "Snippet"
def format_export_name(lang: str, snippet_name: str) -> str:
prefix = LANG_PREFIX[lang]
return f"{prefix}{to_title_case(snippet_name)}"
def snippet_text(lines: List[str]) -> str:
if not lines:
return ""
joined = "\n".join(lines).rstrip("\n")
return (joined + "\n") if joined else ""
def to_js_literal(text: str) -> str:
return json.dumps(text, ensure_ascii=False)
def split_export_prefix(export_name: str) -> Tuple[str | None, str]:
for lang, prefix in LANG_PREFIX.items():
if export_name.startswith(prefix):
suffix = export_name[len(prefix) :]
return lang, suffix or export_name
return None, export_name
def parse_existing_module(path: Path) -> Mapping[str, Mapping[str, SnippetRecord]]:
try:
text = path.read_text(encoding="utf-8")
except FileNotFoundError:
return {}
existing: Dict[str, Dict[str, SnippetRecord]] = {}
for match in EXPORT_RE.finditer(text):
export_name, literal = match.group(1), match.group(2).strip()
lang, snippet_suffix = split_export_prefix(export_name)
if not lang:
continue
try:
decoded = json.loads(literal)
except Exception:
decoded = literal.strip().strip(";")
record = SnippetRecord(
lang=lang,
snippet_name=snippet_suffix or export_name,
export_name=export_name,
text=decoded,
source_rel="(existing module)",
)
existing.setdefault(lang, {})[record.snippet_name] = record
return existing
def iter_source_files(source_dirs: Iterable[Path]) -> Iterator[Path]:
for root in source_dirs:
for path in root.rglob("*"):
if path.is_file() and path.suffix.lower() in LANG_EXTENSIONS:
yield path
def write_if_changed(path: Path, content: str) -> bool:
if path.exists():
try:
current = path.read_text(encoding="utf-8")
if current == content:
return False
except Exception:
pass
ensure_dir(path.parent)
path.write_text(content, encoding="utf-8")
return True
def collect_snippets(
source_dirs: Iterable[Path],
) -> Mapping[str, Mapping[str, Mapping[str, SnippetRecord]]]:
result: Dict[str, Dict[str, Dict[str, SnippetRecord]]] = {}
for file_path in iter_source_files(source_dirs):
lang = detect_language(file_path)
start_re, end_re = markers_for_lang(lang)
try:
text = file_path.read_text(encoding="utf-8")
except Exception as exc: # pragma: no cover
raise RuntimeError(f"Failed to read {file_path}: {exc}") from exc
lines = text.splitlines(keepends=True)
snippets = parse_snippets(lines, start_re, end_re)
if not snippets:
continue
target = normalize_target_name(file_path)
lang_map = result.setdefault(target, {}).setdefault(lang, {})
for snippet_name, content_lines in snippets.items():
if snippet_name in lang_map:
raise ValueError(
f"Duplicate snippet '{snippet_name}' in {file_path} and {lang_map[snippet_name].source_rel}"
)
dedented = dedent(content_lines)
export_name = format_export_name(lang, snippet_name)
source_rel = os.path.relpath(file_path, Path.cwd())
lang_map[snippet_name] = SnippetRecord(
lang=lang,
snippet_name=snippet_name,
export_name=export_name,
text=snippet_text(dedented),
source_rel=source_rel,
)
return result
def render_module(
target: str, lang_map: Mapping[str, Mapping[str, SnippetRecord]]
) -> str:
parts: List[str] = []
parts.append(
"{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */}\n\n"
)
for lang in LANG_ORDER:
snippets = lang_map.get(lang)
if not snippets:
continue
for snippet_name in sorted(snippets.keys()):
record = snippets[snippet_name]
literal = to_js_literal(record.text)
parts.append(f"export const {record.export_name} = {literal};\n\n")
return "".join(parts)
def generate_modules(
snippets_by_target: Mapping[str, Mapping[str, Mapping[str, SnippetRecord]]],
output_root: Path,
) -> None:
modules_written = 0
total_exports = 0
for target in sorted(snippets_by_target.keys()):
module_path = output_root / f"{target}.mdx"
lang_map = snippets_by_target[target]
existing_langs = parse_existing_module(module_path)
for lang, snippets in existing_langs.items():
if lang in lang_map:
continue
print(module_path, " -- ", lang)
merged_lang = lang_map.setdefault(lang, {})
existing_export_names = {rec.export_name for rec in merged_lang.values()}
for snippet_name, record in snippets.items():
# If an export with the same name already exists (e.g., regenerated),
# prefer the newly generated snippet and skip the old one to avoid duplicates.
if record.export_name in existing_export_names:
continue
merged_lang.setdefault(snippet_name, record)
total_exports += sum(len(snippets) for snippets in lang_map.values())
module_content = render_module(target, lang_map)
if write_if_changed(module_path, module_content):
modules_written += 1
print(
f"\nGenerated {total_exports} snippets\n---"
)
def resolve_source_dirs(args_dirs: List[str] | None) -> List[Path]:
if args_dirs:
dirs = [Path(p) for p in args_dirs]
else:
dirs = list(DEFAULT_SOURCE_DIRS)
missing = [str(p) for p in dirs if not p.exists()]
if missing:
raise FileNotFoundError(f"Source directories not found: {', '.join(missing)}")
return dirs
def main() -> int:
parser = argparse.ArgumentParser(
description="Generate MDX snippet modules grouped by test filename."
)
parser.add_argument(
"-s",
"--source-dir",
action="append",
help="Path(s) to language test directories (default: tests/py, ts, rs)",
)
parser.add_argument(
"-o",
"--output-dir",
default=str(Path("docs") / "snippets"),
help="Directory where snippet modules are written (default: docs/snippets)",
)
args = parser.parse_args()
source_dirs = resolve_source_dirs(args.source_dir)
snippets = collect_snippets(source_dirs)
if not snippets:
print("No snippets found.")
return 0
output_root = Path(args.output_dir)
generate_modules(snippets, output_root)
return 0
if __name__ == "__main__":
raise SystemExit(main())
docs/web-tests/py/conftest.py
+56
View File
@@ -0,0 +1,56 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import shutil
from pathlib import Path
import lancedb
import pytest
import pytest_asyncio
from lancedb.db import AsyncConnection, DBConnection
class DatabasePathFactory:
"""Create per-test database directories and ensure they get removed."""
def __init__(self, base_dir: Path) -> None:
self._base_dir = base_dir
self._created: list[Path] = []
def __call__(self, name: str) -> Path:
safe_name = name.replace("/", "_")
path = self._base_dir / safe_name
if path.exists():
shutil.rmtree(path, ignore_errors=True)
path.mkdir(parents=True, exist_ok=True)
self._created.append(path)
return path
def cleanup(self) -> None:
for path in reversed(self._created):
shutil.rmtree(path, ignore_errors=True)
self._created.clear()
@pytest.fixture
def db_path_factory(tmp_path: Path):
factory = DatabasePathFactory(tmp_path)
yield factory
factory.cleanup()
@pytest.fixture
def mem_db() -> DBConnection:
return lancedb.connect("memory://")
@pytest.fixture
def tmp_db(db_path_factory) -> DBConnection:
"""Create a temporary database connection for testing."""
db_path = db_path_factory("tmp_db")
return lancedb.connect(str(db_path))
@pytest_asyncio.fixture
async def mem_db_async() -> AsyncConnection:
return await lancedb.connect_async("memory://")
docs/web-tests/py/test_basic_usage.py
+205
View File
@@ -0,0 +1,205 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
# --8<-- [start:basic_imports]
import json
import lancedb
import pandas as pd
import polars as pl
import pyarrow as pa
# --8<-- [end:basic_imports]
import pytest
data_path = "tests/camelot.json"
def test_basic_usage(db_path_factory):
uri = "./basic_usage_db"
uri = db_path_factory("basic_usage_db")
db = lancedb.connect(uri)
# --8<-- [start:data_load]
with open(data_path, "r") as f:
data = json.load(f)
# --8<-- [end:data_load]
# --8<-- [start:basic_create_table]
table = db.create_table("camelot", data=data, mode="overwrite")
# --8<-- [end:basic_create_table]
assert len(table) == 8
# --8<-- [start:basic_open_table]
table = db.open_table("camelot")
# --8<-- [end:basic_open_table]
# --8<-- [start:basic_create_table_pandas]
pandas_df = pd.DataFrame(data)
table_pd = db.create_table("camelot_pd", data=pandas_df, mode="overwrite")
# --8<-- [end:basic_create_table_pandas]
assert len(table_pd) == 8
db.drop_table("camelot_pd")
# --8<-- [start:basic_create_table_polars]
polars_df = pl.DataFrame(data)
table_pl = db.create_table("camelot_pl", data=polars_df, mode="overwrite")
# --8<-- [end:basic_create_table_polars]
assert len(table_pl) == 8
db.drop_table("camelot_pl")
# --8<-- [start:basic_create_empty_table]
schema = pa.schema(
[
pa.field("id", pa.uint16()),
pa.field("name", pa.string()),
pa.field("role", pa.string()),
pa.field("description", pa.string()),
pa.field("vector", pa.list_(pa.float32(), 4)),
pa.field(
"stats",
pa.struct(
[
pa.field("strength", pa.int8()),
pa.field("courage", pa.int8()),
pa.field("magic", pa.int8()),
pa.field("wisdom", pa.int8()),
]
),
),
]
)
db.create_table("camelot_pa", schema=schema, mode="overwrite")
# --8<-- [end:basic_create_empty_table]
assert "camelot_pa" in db.list_tables().tables
db.drop_table("camelot_pa")
# --8<-- [start:basic_add_data]
magical_characters = [
{
"id": 9,
"name": "Morgan le Fay",
"role": "Sorceress",
"description": "A powerful enchantress, Arthur's half-sister, and a complex figure who oscillates between aiding and opposing Camelot.",
"vector": [0.10, 0.84, 0.25, 0.70],
"stats": { "strength": 2, "courage": 3, "magic": 5, "wisdom": 4 }
},
{
"id": 10,
"name": "The Lady of the Lake",
"role": "Mystical Guardian",
"description": "A mysterious supernatural figure associated with Avalon, known for giving Arthur the sword Excalibur.",
"vector": [0.00, 0.90, 0.58, 0.88],
"stats": { "strength": 2, "courage": 3, "magic": 5, "wisdom": 5 }
}
]
table.add(magical_characters)
# --8<-- [end:basic_add_data]
assert len(table) == 10
# --8<-- [start:basic_vector_search]
query_vector = [0.03, 0.85, 0.61, 0.90]
result = table.search(query_vector).limit(5).to_polars()
print(result)
# --8<-- [end:basic_vector_search]
# --8<-- [start:basic_add_columns]
table.add_columns(
{
"power": "cast(((stats.strength + stats.courage + stats.magic + stats.wisdom) / 4.0) as float)"
}
)
# --8<-- [end:basic_add_columns]
assert "power" in table.schema.names
# Run examples to illustrate search
# --8<-- [start:basic_vector_search_q1]
# Who are the characters similar to "wizard"?
query_vector_1 = [0.03, 0.85, 0.61, 0.90]
r1 = (
table.search(query_vector_1)
.limit(5)
.select(["name", "role", "description"])
.to_polars()
)
print(r1)
# --8<-- [end:basic_vector_search_q1]
# --8<-- [start:basic_vector_search_q2]
# Who are the characters with high magic stats?
query_vector_2 = [0.03, 0.85, 0.61, 0.90]
r2 = (
table.search(query_vector_2)
.where("stats.magic > 3")
.select(["name", "role", "description"])
.limit(5)
.to_polars()
)
print(r2)
# --8<-- [end:basic_vector_search_q2]
# --8<-- [start:basic_vector_search_q3]
# Who are the strongest characters?
r3 = (
table.search()
.where("stats.strength > 3")
.select(["name", "role", "description"])
.limit(5)
.to_polars()
)
print(r3)
# --8<-- [end:basic_vector_search_q3]
# --8<-- [start:basic_vector_search_q4]
# Who are the strongest characters?
r4 = (
table.search()
.select(["name", "role", "description", "power"])
.to_polars()
)
print(r4)
# --8<-- [end:basic_vector_search_q4]
# --8<-- [start:basic_sort_polars]
# Sort Polars DataFrame by power in descending order
print(r4.sort("power", descending=True).limit(5))
# --8<-- [end:basic_sort_polars]
sorted_power = r4.sort("power", descending=True)["power"].to_list()
assert sorted_power == sorted(sorted_power, reverse=True)
assert sorted_power[0] == 4.0
# --8<-- [start:basic_drop_columns]
table.drop_columns(["power"])
# --8<-- [end:basic_drop_columns]
# --8<-- [start:basic_delete_rows]
table.delete('role = "Traitor Knight"')
# --8<-- [end:basic_delete_rows]
assert len(table) == 9
# --8<-- [start:basic_drop_table]
db.drop_table("camelot")
# --8<-- [end:basic_drop_table]
assert "camelot" not in db.list_tables().tables
@pytest.mark.asyncio
async def test_basic_usage_async_api(db_path_factory):
uri = db_path_factory("basic_usage_async_db")
with open(data_path, "r") as f:
data = json.load(f)
# --8<-- [start:basic_async_api]
import lancedb
async_db = await lancedb.connect_async(uri)
async_table = await async_db.create_table(
"camelot_async",
data=data,
mode="overwrite",
)
query_vector = [0.03, 0.85, 0.61, 0.90]
async_results = await (
await async_table.search(query_vector)
).limit(5).select(["name", "role", "description"]).to_polars()
print(async_results)
# --8<-- [end:basic_async_api]
assert async_results.height >= 1
docs/web-tests/py/test_build_with_ai_agents.py
+109
View File
@@ -0,0 +1,109 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
# --8<-- [start:camelot_schema]
from lancedb.pydantic import LanceModel
from pydantic import ConfigDict
class Stats(LanceModel):
model_config = ConfigDict(strict=True, extra="forbid")
strength: int
courage: int
magic: int
wisdom: int
class Character(LanceModel):
model_config = ConfigDict(strict=True, extra="forbid")
id: int
name: str
role: str
description: str
stats: Stats
image_filename: str
image: bytes
# --8<-- [end:camelot_schema]
# --8<-- [start:camelot_batches]
import json
from collections.abc import Iterator
from pathlib import Path
def validated_batches(
input_path: Path, batch_size: int
) -> Iterator[list[dict]]:
raw_records = json.loads(input_path.read_text())
asset_root = input_path.parent.parent
batch: list[dict] = []
for raw in raw_records:
payload = dict(raw)
image_path = asset_root / payload.pop("img")
payload["image_filename"] = image_path.name
payload["image"] = image_path.read_bytes()
character = Character.model_validate(payload)
batch.append(character.model_dump(mode="python"))
if len(batch) == batch_size:
yield batch
batch = []
if batch:
yield batch
# --8<-- [end:camelot_batches]
# --8<-- [start:camelot_oss_ingestion]
import lancedb
def ingest_oss(
input_path: Path,
uri: str = "data/camelot.lancedb",
table_name: str = "camelot_multimodal",
batch_size: int = 64,
):
db = lancedb.connect(uri)
if table_name in db.list_tables():
raise ValueError(
f"Table {table_name!r} already exists. Choose a fresh table name."
)
table = db.create_table(table_name, schema=Character)
for batch in validated_batches(input_path, batch_size):
table.add(batch)
table.optimize()
return table
if __name__ == "__main__":
ingest_oss(Path("data/camelot.json"))
# --8<-- [end:camelot_oss_ingestion]
def test_camelot_oss_ingestion(tmp_path):
source = Path(
"docs/web/static/assets/tutorials/build-with-ai-agents/camelot/data/camelot.json"
)
table = ingest_oss(
source,
uri=str(tmp_path / "camelot.lancedb"),
table_name="camelot",
)
rows = (
table.search()
.select(["id", "name", "image", "image_filename"])
.limit(8)
.to_list()
)
assert len(rows) == 8
assert all(row["image"] for row in rows)
assert all(row["image_filename"].endswith(".jpg") for row in rows)
docs/web-tests/py/test_connection.py
+148
View File
@@ -0,0 +1,148 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
from pathlib import Path
import shutil
def test_connection():
# --8<-- [start:connect]
import lancedb
uri = "ex_lancedb"
db = lancedb.connect(uri)
# --8<-- [end:connect]
assert db is not None
shutil.rmtree(uri, ignore_errors=True)
assert not Path(uri).exists()
async def connect_async_example():
# --8<-- [start:connect_async]
import lancedb
uri = "ex_lancedb"
async_db = await lancedb.connect_async(uri)
# --8<-- [end:connect_async]
return async_db
def connect_enterprise_quickstart_config():
import lancedb
# --8<-- [start:connect_enterprise_quickstart]
uri = "db://your-database-uri"
api_key = "your-api-key"
region = "us-east-1"
host_override = "https://your-enterprise-endpoint.com"
db = lancedb.connect(
uri=uri,
api_key=api_key,
region=region,
host_override=host_override,
)
# --8<-- [end:connect_enterprise_quickstart]
return db
def test_connect_enterprise_quickstart(monkeypatch):
import lancedb
captured = {}
def fake_connect(**kwargs):
captured.update(kwargs)
return object()
monkeypatch.setattr(lancedb, "connect", fake_connect)
db = connect_enterprise_quickstart_config()
assert db is not None
assert captured == {
"uri": "db://your-database-uri",
"api_key": "your-api-key",
"region": "us-east-1",
"host_override": "https://your-enterprise-endpoint.com",
}
def connect_object_storage_config():
# --8<-- [start:connect_object_storage]
import lancedb
uri = "s3://your-bucket/path"
# You can also use "gs://your-bucket/path" or "az://your-container/path".
db = lancedb.connect(uri)
# --8<-- [end:connect_object_storage]
return db
def namespace_table_ops_example():
# --8<-- [start:namespace_table_ops]
import lancedb
db = lancedb.connect_namespace("dir", {"root": "./local_lancedb"})
# Create namespace tree: prod/search
db.create_namespace(["prod"], mode="exist_ok")
db.create_namespace(["prod", "search"], mode="exist_ok")
db.create_namespace(["prod", "recommendations"], mode="exist_ok")
db.create_table(
"user",
data=[{"id": 1, "vector": [0.1, 0.2], "name": "alice"}],
namespace_path=["prod", "search"],
mode="create", # use "overwrite" only if you want to replace existing table
)
db.create_table(
"user",
data=[{"id": 2, "vector": [0.3, 0.4], "name": "bob"}],
namespace_path=["prod", "recommendations"],
mode="create", # use "overwrite" only if you want to replace existing table
)
# Verify
print(db.list_namespaces()) # ['prod']
print(db.list_namespaces(namespace_path=["prod"])) # ['recommendations', 'search']
print(db.list_tables(namespace_path=["prod", "search"])) # ['user']
print(db.list_tables(namespace_path=["prod", "recommendations"])) # ['user']
# --8<-- [end:namespace_table_ops]
def namespace_admin_ops_example():
# --8<-- [start:namespace_admin_ops]
import lancedb
db = lancedb.connect_namespace("dir", {"root": "./local_lancedb"})
namespace = ["prod", "search"]
db.create_namespace(["prod"])
db.create_namespace(["prod", "search"])
child_namespaces = db.list_namespaces(namespace_path=["prod"]).namespaces
print(f"Child namespaces under {namespace}: {child_namespaces}")
# Child namespaces under ['prod', 'search']: ['search']
metadata = db.describe_namespace(["prod", "search"])
print(f"Metadata for namespace {namespace}: {metadata}")
# Metadata for namespace ['prod', 'search']: properties=None
db.drop_namespace(["prod", "search"], mode="skip")
db.drop_namespace(["prod"], mode="skip")
# --8<-- [end:namespace_admin_ops]
return child_namespaces, metadata
async def connect_object_storage_config_async():
# --8<-- [start:connect_object_storage_async]
import lancedb
uri = "s3://your-bucket/path"
# You can also use "gs://your-bucket/path" or "az://your-container/path".
async_db = await lancedb.connect_async(uri)
# --8<-- [end:connect_object_storage_async]
return async_db
docs/web-tests/py/test_doc_function_examples.py
+34
View File
@@ -0,0 +1,34 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
from lancedb.index import FTS
def test_python_boolean_query_operators(tmp_db):
table = tmp_db.create_table(
"boolean_docs",
[
{"id": 1, "text": "puppy"},
{"id": 2, "text": "merrily"},
{"id": 3, "text": "puppy merrily"},
{"id": 4, "text": "other"},
],
)
table.create_index("text", config=FTS())
# --8<-- [start:fts_boolean_queries]
from lancedb.query import MatchQuery
intersection = MatchQuery("puppy", "text") & MatchQuery("merrily", "text")
union = MatchQuery("puppy", "text") | MatchQuery("merrily", "text")
# --8<-- [end:fts_boolean_queries]
def ids(query):
return set(table.search(query, query_type="fts").to_arrow()["id"].to_pylist())
assert ids(intersection) == {3}
assert ids(union) == {1, 2, 3}
a = MatchQuery("puppy", "text")
b = MatchQuery("merrily", "text")
assert ids(a and b) == {2, 3}
assert ids(a or b) == {1, 3}
docs/web-tests/py/test_doc_storage_examples.py
+84
View File
@@ -0,0 +1,84 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import lancedb
import numpy as np
import pyarrow as pa
import pytest
def test_rebuild_vector_index(tmp_db):
rng = np.random.default_rng(7)
table = tmp_db.create_table(
"rebuild_index",
[{"id": i, "vector": v.tolist()} for i, v in enumerate(rng.random((512, 4)))],
)
table.create_index(
"vector", config=lancedb.index.IvfFlat(distance_type="l2", num_partitions=4)
)
original_index = table.list_indices()[0]
table.add(
[
{"id": 512 + i, "vector": v.tolist()}
for i, v in enumerate(rng.random((64, 4)) + 5)
]
)
assert table.index_stats("vector_idx").num_unindexed_rows == 64
# --8<-- [start:storage_rebuild_vector_index]
from lancedb.index import IvfFlat
table.create_index(
"vector",
config=IvfFlat(distance_type="l2", num_partitions=4),
replace=True,
)
# --8<-- [end:storage_rebuild_vector_index]
indices = table.list_indices()
assert len(indices) == 1
assert indices[0].name == original_index.name
assert indices[0].index_uuid != original_index.index_uuid
stats = table.index_stats("vector_idx")
assert stats.num_indexed_rows == 576
assert stats.num_unindexed_rows == 0
def test_local_shallow_clone_pins_source_version(tmp_path):
db = lancedb.connect(str(tmp_path))
source = db.create_table("source", [{"id": 1}])
clone = db.clone_table("snapshot", source_uri=str(tmp_path / "source.lance"))
source.add([{"id": 2}])
assert source.count_rows() == 2
assert clone.to_arrow().to_pylist() == [{"id": 1}]
def _reader():
return pa.RecordBatchReader.from_batches(
pa.schema([("id", pa.int64())]),
[pa.record_batch([[1, 2]], names=["id"]), pa.record_batch([[3]], names=["id"])],
)
@pytest.mark.parametrize("parallelism", [1, 2])
def test_streaming_write_parallelism(tmp_db, parallelism):
table = tmp_db.create_table("streaming", [{"id": 0}])
progress = []
table.add(_reader(), write_parallelism=parallelism, progress=progress.append)
assert sorted(table.to_arrow()["id"].to_pylist()) == [0, 1, 2, 3]
assert progress[-1]["done"]
assert progress[-1]["output_rows"] == 3
assert {p["total_tasks"] for p in progress} == {parallelism}
@pytest.mark.asyncio
@pytest.mark.parametrize("parallelism", [1, 2])
async def test_async_streaming_write_parallelism(mem_db_async, parallelism):
table = await mem_db_async.create_table("streaming", [{"id": 0}])
progress = []
await table.add(_reader(), write_parallelism=parallelism, progress=progress.append)
data = await table.to_arrow()
assert sorted(data["id"].to_pylist()) == [0, 1, 2, 3]
assert progress[-1]["done"]
assert progress[-1]["output_rows"] == 3
assert {p["total_tasks"] for p in progress} == {parallelism}
docs/web-tests/py/test_embedding.py
+141
View File
@@ -0,0 +1,141 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import lancedb
# --8<-- [start:imports]
from lancedb.pydantic import LanceModel, Vector
from lancedb.embeddings import get_registry
# --8<-- [end:imports]
import pytest
@pytest.mark.skip(reason="OpenAI is not available in the test environment")
def test_create_embedding_function():
# --8<-- [start:create_embedding_function]
func = get_registry().get("openai").create(
name="text-embedding-3-small",
max_retries=7,
)
# --8<-- [end:create_embedding_function]
assert func is not None
@pytest.mark.skip(reason="OpenAI is not available in the test environment")
def test_embeddings_openai():
# --8<-- [start:openai_embeddings]
db = lancedb.connect("/tmp/db")
func = get_registry().get("openai").create(name="text-embedding-ada-002")
class Words(LanceModel):
text: str = func.SourceField()
vector: Vector(func.ndims()) = func.VectorField()
table = db.create_table("words", schema=Words, mode="overwrite")
table.add([{"text": "hello world"}, {"text": "goodbye world"}])
query = "greetings"
actual = table.search(query).limit(1).to_pydantic(Words)[0]
print(actual.text)
# --8<-- [end:openai_embeddings]
@pytest.mark.skip(reason="OpenAI is not available in the test environment")
@pytest.mark.asyncio
async def test_embeddings_openai_async():
uri = "memory://"
# --8<-- [start:async_openai_embeddings]
db = await lancedb.connect_async(uri)
func = get_registry().get("openai").create(name="text-embedding-ada-002")
class Words(LanceModel):
text: str = func.SourceField()
vector: Vector(func.ndims()) = func.VectorField()
table = await db.create_table("words", schema=Words, mode="overwrite")
await table.add([{"text": "hello world"}, {"text": "goodbye world"}])
query = "greetings"
actual = await (await table.search(query)).limit(1).to_pydantic(Words)[0]
print(actual.text)
# --8<-- [end:async_openai_embeddings]
@pytest.mark.skip(reason="OpenAI is not available in the test environment")
def test_embeddings_manual_query():
# --8<-- [start:manual_query_embeddings]
db = lancedb.connect("/tmp/db")
func = get_registry().get("openai").create(name="text-embedding-ada-002")
class Words(LanceModel):
text: str = func.SourceField()
vector: Vector(func.ndims()) = func.VectorField()
table = db.create_table("words", schema=Words, mode="overwrite")
table.add([{"text": "hello world"}, {"text": "goodbye world"}])
query_vector = func.generate_embeddings(["greetings"])[0]
# --8<-- [start:manual_query_search]
# query_vector is assumed to already be generated by your embedding function
actual = table.search(query_vector).limit(1).to_pydantic(Words)[0]
print(actual.text)
# --8<-- [end:manual_query_search]
# --8<-- [end:manual_query_embeddings]
def test_custom_embedding_function():
# --8<-- [start:embedding_function]
from functools import cached_property
from lancedb.embeddings import TextEmbeddingFunction, register
class MyEmbeddingModel:
def __init__(self, model_name: str):
self.model_name = model_name
def encode(self, texts: list[str]) -> list[list[float]]:
return [[1.0, 2.0, 3.0] for _ in texts]
@register("my-embedder")
class MyTextEmbedder(TextEmbeddingFunction):
model_name: str = "my-model"
def generate_embeddings(self, texts: list[str]) -> list[list[float]]:
# Your embedding logic here
return self._model.encode(texts)
def ndims(self) -> int:
# Return the dimensionality of the embeddings
return len(self.generate_embeddings(["test"])[0])
@cached_property
def _model(self) -> MyEmbeddingModel:
# Initialize your model once
return MyEmbeddingModel(self.model_name)
# --8<-- [end:embedding_function]
def test_embeddings_secret():
# --8<-- [start:register_secret]
registry = get_registry()
registry.set_var("api_key", "sk-...")
func = registry.get("openai").create(api_key="$var:api_key")
# --8<-- [end:register_secret]
try:
import torch
except ImportError:
pytest.skip("torch not installed")
# --8<-- [start:register_device]
import torch
registry = get_registry()
if torch.cuda.is_available():
registry.set_var("device", "cuda")
func = registry.get("huggingface").create(device="$var:device:cpu")
# --8<-- [end:register_device]
assert func.device == "cuda" if torch.cuda.is_available() else "cpu"
docs/web-tests/py/test_function.py
+242
View File
@@ -0,0 +1,242 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""Examples for docs/web/management/function.mdx.
Registering a Function and filling its column need a LanceDB Enterprise
deployment, so `test_function_enterprise` runs only when the connection
variables the Enterprise examples already use are set: LANCEDB_URI,
LANCEDB_API_KEY and LANCEDB_HOST_OVERRIDE, plus LANCEDB_REGION if the
deployment needs one. The walkthrough calls its table `function_demo` and its
Function `double`; the test creates both under names unique to its run, never
replaces an existing object, and drops only what it created.
The other tests run locally and check the client side of the same code: what
`@udf` sends for registration, how a registered version binds a column, and
where a local connection stops. They do not run a Function remotely.
"""
import json
import os
import sys
import uuid
from pathlib import Path
import lancedb
import pyarrow as pa
import pytest
from lancedb import col
from lancedb.functions import FunctionApplication, FunctionVersion
# --8<-- [start:function_define]
from lancedb import udf
@udf
def double(value: float) -> float:
return value * 2
# --8<-- [end:function_define]
# The definition the walkthrough registers. The Enterprise test registers a copy
# of it under a name that only its run uses.
WALKTHROUGH_DOUBLE = double
def describe(value: float):
return {"half": value / 2, "note": None}
# A Function version as the service returns it after registration: the SDK's
# shared golden reply, with the name and signature of the definition above.
SERVICE_REPLY = (
Path(__file__).parents[3]
/ "rust"
/ "lancedb"
/ "tests"
/ "fixtures"
/ "first_class_functions"
/ "v1"
/ "remote_function_job.json"
)
def registered_version(definition) -> FunctionVersion:
request = definition.registration_request
reply = json.loads(SERVICE_REPLY.read_text())["result"]
reply["name"] = request.name
reply["signature"] = json.loads(request.signature.to_canonical_json())
return FunctionVersion.from_json(json.dumps(reply))
def require_env(var_name: str) -> str:
"""Skip the test unless the required environment variable is provided."""
value = os.environ.get(var_name)
if not value:
pytest.skip(f"Set {var_name} to run this Enterprise example")
return value
def test_function_runs_locally_as_python():
# --8<-- [start:function_local_call]
print(double(1.5))
# 3.0
# --8<-- [end:function_local_call]
assert double(1.5) == 3.0
def test_function_registration_request():
request = double.registration_request
assert request.name == "double"
assert [
(parameter.name, parameter.arrow_type, parameter.nullable)
for parameter in request.signature.inputs
] == [("value", "float64", False)]
output = request.signature.output
assert (output.kind, output.arrow_type, output.nullable) == (
"scalar",
"float64",
False,
)
version = request.runtime.python_version
assert version == f"{sys.version_info.major}.{sys.version_info.minor}"
assert request.runtime.environment.packages == ()
def test_function_output_nullability():
inputs = pa.schema([pa.field("value", pa.float64(), nullable=False)])
# A named-struct output may declare nullable fields.
fields = [
pa.field("half", pa.float64(), nullable=False),
pa.field("note", pa.string(), nullable=True),
]
struct = udf(input_schema=inputs, output_schema=pa.schema(fields))(describe)
output = struct.registration_request.signature.output
assert output.kind == "named_struct"
assert [(field.name, field.nullable) for field in output.fields] == [
("half", False),
("note", True),
]
# A scalar output, and a struct declared as one nullable field, may not be.
for nullable in (
pa.field("result", pa.float64(), nullable=True),
pa.field("result", pa.struct(fields), nullable=True),
):
with pytest.raises(ValueError, match="non-nullable"):
udf(input_schema=inputs, output_schema=nullable)(describe)
def test_function_registration_is_remote(tmp_path):
db = lancedb.connect(tmp_path)
with pytest.raises(NotImplementedError):
db.create_function(double)
def test_function_version_binds_a_column(tmp_path):
version = registered_version(double)
application = version(value=col("value"))
assert isinstance(application, FunctionApplication)
assert application.function.name == "double"
assert application.function.version == version.version
assert [
(binding.parameter, binding.kind, binding.value["path"])
for binding in application.inputs
] == [("value", "column", "value")]
with pytest.raises(TypeError, match="direct col"):
version(value=1.5)
with pytest.raises(TypeError, match="missing"):
version(x=col("value"))
# The binding is accepted by add_columns, which a local table refuses:
# a Function column needs LanceDB Cloud or Enterprise.
table = lancedb.connect(tmp_path).create_table("t", [{"value": 1.0}])
with pytest.raises(NotImplementedError, match="Cloud and Enterprise"):
table.add_columns({"doubled": application})
def test_function_enterprise(monkeypatch):
require_env("LANCEDB_URI")
require_env("LANCEDB_API_KEY")
require_env("LANCEDB_HOST_OVERRIDE")
# --8<-- [start:function_connect]
import os
import lancedb
db = lancedb.connect(
uri=os.environ["LANCEDB_URI"], # db://your-database
api_key=os.environ["LANCEDB_API_KEY"],
region=os.environ.get("LANCEDB_REGION", "us-east-1"),
host_override=os.environ["LANCEDB_HOST_OVERRIDE"],
)
# --8<-- [end:function_connect]
# In a shared database `function_demo` or `double` may already exist, so
# this run creates both under names only it uses, and drops only those.
run = uuid.uuid4().hex[:12]
table_name = f"function_demo_{run}"
double = udf(name=f"double_{run}")(WALKTHROUGH_DOUBLE.__wrapped__)
create_table = db.create_table
def create_run_table(name, *args, **kwargs):
assert name == "function_demo"
return create_table(table_name, *args, **kwargs)
monkeypatch.setattr(db, "create_table", create_run_table)
table = None
version = None
try:
# --8<-- [start:function_create_table]
import pyarrow as pa
schema = pa.schema([pa.field("value", pa.float64(), nullable=False)])
table = db.create_table(
"function_demo",
[{"value": 1.0}, {"value": 2.0}, {"value": 3.0}],
schema=schema,
)
# --8<-- [end:function_create_table]
# --8<-- [start:function_register]
job = db.create_function_async(double)
version = job.wait()
# --8<-- [end:function_register]
assert version.name == f"double_{run}"
# --8<-- [start:function_apply]
from lancedb import col
table.add_columns({"doubled": version(value=col("value"))})
# --8<-- [end:function_apply]
# --8<-- [start:function_refresh]
refresh = table.refresh_column_async("doubled")
result = refresh.wait()
print(result.rows_assigned)
# 3
# --8<-- [end:function_refresh]
assert result.rows_assigned == 3
assert result.rows_failed == 0
# --8<-- [start:function_inspect]
rows = table.search().select(["value", "doubled"]).to_list()
print(sorted((row["value"], row["doubled"]) for row in rows))
# [(1.0, 2.0), (2.0, 4.0), (3.0, 6.0)]
# --8<-- [end:function_inspect]
assert sorted((row["value"], row["doubled"]) for row in rows) == [
(1.0, 2.0),
(2.0, 4.0),
(3.0, 6.0),
]
finally:
if table is not None:
db.drop_table(table_name)
if version is not None:
db.drop_function(version.name, version=version.version)
docs/web-tests/py/test_indexing.py
+757
View File
@@ -0,0 +1,757 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import random
import string
import uuid
import numpy as np
import pyarrow as pa
import pytest
def _make_vector_rows(count: int, dim: int, column: str = "vector"):
rows = [
{column: np.random.random(dim).astype(np.float32).tolist(), "id": i}
for i in range(count)
]
assert len(rows) == count
assert all(len(row[column]) == dim for row in rows)
return rows
def test_vector_index_configure_ivf(tmp_db):
table = tmp_db.create_table(
"vector_index_configure_ivf",
_make_vector_rows(512, 4),
mode="overwrite",
)
# --8<-- [start:vector_index_configure_ivf]
table.create_index(metric="l2", num_partitions=16, num_sub_vectors=4)
# --8<-- [end:vector_index_configure_ivf]
assert table.list_indices()
def test_vector_index_setup(tmp_db):
tmp_db.create_table(
"vector-index-tbl",
_make_vector_rows(8, 4),
mode="overwrite",
)
db = tmp_db
# --8<-- [start:vector_index_setup]
table_name = "vector-index-tbl"
table = db.open_table(table_name)
# --8<-- [end:vector_index_setup]
assert table.name == table_name
def test_vector_index_build_ivf(tmp_db):
table = tmp_db.create_table(
"vector-index-build-ivf",
_make_vector_rows(512, 4, column="keywords_embeddings"),
mode="overwrite",
)
db = tmp_db
# --8<-- [start:vector_index_build_ivf]
table_name = "vector-index-build-ivf"
table = db.open_table(table_name)
table.create_index(
metric="cosine",
vector_column_name="keywords_embeddings",
)
# --8<-- [end:vector_index_build_ivf]
assert table.list_indices()
def test_vector_index_nested_field(tmp_db):
dim = 2
schema = pa.schema(
[
pa.field("id", pa.int32()),
pa.field(
"image",
pa.struct([pa.field("embedding", pa.list_(pa.float32(), dim))]),
),
]
)
data = [
{
"id": i,
"image": {"embedding": np.random.random(dim).astype(np.float32).tolist()},
}
for i in range(512)
]
table = tmp_db.create_table(
"vector_index_nested_field", data=data, schema=schema, mode="overwrite"
)
# --8<-- [start:vector_index_nested_field]
# The vector column `embedding` is nested inside the `image` struct.
# Pass its full dotted path as `vector_column_name`; the same path is used
# at query time and is what `list_indices()` reports under `columns`.
table.create_index(
vector_column_name="image.embedding",
num_partitions=1,
num_sub_vectors=1,
name="image_embedding_idx",
)
results = (
table.search([0.0, 1.0], vector_column_name="image.embedding")
.limit(1)
.to_list()
)
# --8<-- [end:vector_index_nested_field]
assert table.index_stats("image_embedding_idx")
assert len(results) == 1
@pytest.mark.asyncio
async def test_vector_index_async_config(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
# --8<-- [start:vector_index_async_config]
import lancedb
import numpy as np
from lancedb.index import IvfPq
async def main():
data = [
{"id": i, "vector": np.random.random(8).astype(np.float32).tolist()}
for i in range(512)
]
db = await lancedb.connect_async("ex_lancedb")
table = await db.create_table(
"vector_index_async", data=data, mode="overwrite"
)
await table.create_index(
"vector",
config=IvfPq(
distance_type="cosine",
num_partitions=16,
num_sub_vectors=4,
),
)
return await table.list_indices()
# --8<-- [end:vector_index_async_config]
assert await main()
def test_vector_index_query_ivf(tmp_db):
dim = 1536
data = [
{"id": i, "keywords_embeddings": np.random.random(dim).tolist()}
for i in range(512)
]
table = tmp_db.create_table("vector_index_query_ivf", data, mode="overwrite")
table.create_index(
metric="cosine",
vector_column_name="keywords_embeddings",
)
# --8<-- [start:vector_index_query_ivf]
tbl = table
tbl.search(np.random.random((1536))).limit(2).nprobes(20).refine_factor(
10
).to_pandas()
# --8<-- [end:vector_index_query_ivf]
df = (
table.search(np.random.random((1536)))
.limit(2)
.nprobes(20)
.refine_factor(10)
.to_pandas()
)
assert len(df) == 2
def test_vector_index_nprobes(tmp_db):
dim = 128
data = [
{"id": i, "keywords_embeddings": np.random.random(dim).tolist()}
for i in range(512)
]
table = tmp_db.create_table("vector_index_nprobes", data, mode="overwrite")
table.create_index(
metric="cosine",
vector_column_name="keywords_embeddings",
)
# --8<-- [start:vector_index_nprobes]
# Always scan 10 partitions; scan up to 50 only if the initial pass
# returns fewer than `limit` results (common with narrow filters).
(
table.search(np.random.random(128))
.minimum_nprobes(10)
.maximum_nprobes(50)
.where("id > 100")
.limit(5)
.to_pandas()
)
# --8<-- [end:vector_index_nprobes]
def test_vector_index_distance_range(tmp_db):
dim = 128
data = [
{"id": i, "keywords_embeddings": np.random.random(dim).tolist()}
for i in range(256)
]
table = tmp_db.create_table("vector_index_distance_range", data, mode="overwrite")
table.create_index(
metric="cosine",
vector_column_name="keywords_embeddings",
)
# --8<-- [start:vector_index_distance_range]
# Only return results whose distance falls within [0.0, 0.5).
# Useful for near-duplicate detection or thresholded similarity search.
(
table.search(np.random.random(128))
.distance_range(lower_bound=0.0, upper_bound=0.5)
.limit(10)
.to_pandas()
)
# --8<-- [end:vector_index_distance_range]
def test_vector_index_bypass_recall(tmp_db):
dim = 128
data = [
{"id": i, "keywords_embeddings": np.random.random(dim).tolist()}
for i in range(256)
]
table = tmp_db.create_table("vector_index_bypass_recall", data, mode="overwrite")
table.create_index(
metric="cosine",
vector_column_name="keywords_embeddings",
)
# --8<-- [start:vector_index_bypass_recall]
query = np.random.random(128)
k = 10
# Ground truth: flat (exhaustive) scan, ignoring the ANN index.
truth = set(table.search(query).bypass_vector_index().limit(k).to_pandas()["id"])
# ANN results with the current nprobes setting.
ann = set(table.search(query).nprobes(20).limit(k).to_pandas()["id"])
recall_at_k = len(truth & ann) / k
# --8<-- [end:vector_index_bypass_recall]
assert 0.0 <= recall_at_k <= 1.0
def test_vector_index_custom_name(tmp_db):
table = tmp_db.create_table(
"vector_index_custom_name",
_make_vector_rows(512, 8, column="keywords_embeddings"),
mode="overwrite",
)
# --8<-- [start:vector_index_custom_name]
# Override the default `{column}_idx` convention by passing `name=...`.
table.create_index(
metric="cosine",
vector_column_name="keywords_embeddings",
name="my_custom_index",
)
table.wait_for_index(["my_custom_index"])
print(table.index_stats("my_custom_index"))
# --8<-- [end:vector_index_custom_name]
assert table.index_stats("my_custom_index")
def test_vector_index_hnsw(tmp_db):
table = tmp_db.create_table(
"vector_index_hnsw",
_make_vector_rows(64, 16),
mode="overwrite",
)
# --8<-- [start:vector_index_build_hnsw]
table.create_index(index_type="IVF_HNSW_SQ")
# --8<-- [end:vector_index_build_hnsw]
# --8<-- [start:vector_index_query_hnsw]
tbl = table
tbl.search(np.random.random((16))).limit(2).to_pandas()
# --8<-- [end:vector_index_query_hnsw]
df = table.search(np.random.random((16))).limit(2).to_pandas()
assert len(df) == 2
def test_vector_index_binary(tmp_db):
table_name = "hamming-index-tbl"
ndim = 256
schema = pa.schema(
[
pa.field("id", pa.int64()),
pa.field("vector", pa.list_(pa.uint8(), ndim // 8)),
]
)
# --8<-- [start:vector_index_binary_schema]
table = tmp_db.create_table(table_name, schema=schema, mode="overwrite")
# --8<-- [end:vector_index_binary_schema]
data = []
for i in range(64):
vector = np.random.randint(0, 2, size=ndim)
vector = np.packbits(vector)
data.append({"id": i, "vector": vector})
# --8<-- [start:vector_index_binary_add_data]
table.add(data)
# --8<-- [end:vector_index_binary_add_data]
# --8<-- [start:vector_index_binary_build_index]
table.create_index(
metric="hamming",
vector_column_name="vector",
index_type="IVF_FLAT",
)
# --8<-- [end:vector_index_binary_build_index]
# --8<-- [start:vector_index_binary_search]
query = np.random.randint(0, 2, size=ndim)
query = np.packbits(query)
df = table.search(query).metric("hamming").limit(10).to_pandas()
df.vector = df.vector.apply(np.unpackbits)
# --8<-- [end:vector_index_binary_search]
assert not df.empty
def test_quantization_custom_params(tmp_db):
table = tmp_db.create_table(
"quantization-custom-params",
_make_vector_rows(256, 64),
mode="overwrite",
)
# --8<-- [start:quantization_custom_params]
table.create_index(
index_type="IVF_RQ",
num_bits=2,
max_iterations=100,
sample_rate=512,
)
# --8<-- [end:quantization_custom_params]
assert table.list_indices()
def test_vector_index_check_status(tmp_db):
table = tmp_db.create_table(
"vector_index_check_status",
_make_vector_rows(512, 8, column="keywords_embeddings"),
mode="overwrite",
)
table.create_index(
metric="cosine",
vector_column_name="keywords_embeddings",
)
# --8<-- [start:vector_index_check_status]
index_name = "keywords_embeddings_idx"
table.wait_for_index([index_name])
print(table.index_stats(index_name))
# --8<-- [end:vector_index_check_status]
assert table.index_stats(index_name)
def test_scalar_index_build(tmp_db):
table = tmp_db.create_table(
"scalar_index_build",
[
{"book_id": 1, "publisher": "A", "vector": [0.1, 0.2]},
{"book_id": 2, "publisher": "B", "vector": [0.2, 0.3]},
],
mode="overwrite",
)
db = tmp_db
# --8<-- [start:scalar_index_build]
tbl = db.open_table("scalar_index_build")
tbl.create_scalar_index("book_id")
tbl.create_scalar_index("publisher", index_type="BITMAP")
# --8<-- [end:scalar_index_build]
assert tbl.list_indices()
def test_scalar_index_wait(tmp_db):
table = tmp_db.create_table(
"scalar_index_wait",
[{"label": "fiction"}],
mode="overwrite",
)
table.create_scalar_index("label")
# --8<-- [start:scalar_index_wait]
index_name = "label_idx"
table.wait_for_index([index_name])
# --8<-- [end:scalar_index_wait]
assert table.list_indices()
def test_scalar_index_optimize(tmp_db):
table = tmp_db.create_table(
"scalar_index_optimize",
[{"vector": [7.0, 8.0], "book_id": 3}],
mode="overwrite",
)
# --8<-- [start:scalar_index_optimize]
table.add([{"vector": [7, 8], "book_id": 4}])
table.optimize()
# --8<-- [end:scalar_index_optimize]
result = table.search().where("book_id = 4").limit(10).to_pandas()
assert len(result) == 1
def test_scalar_index_filter(tmp_db):
table = tmp_db.create_table(
"books",
[
{"vector": [1.1, 1.2], "book_id": 1},
{"vector": [2.1, 2.2], "book_id": 2},
],
mode="overwrite",
)
db = tmp_db
# --8<-- [start:scalar_index_filter]
table = db.open_table("books")
result = table.search().where("book_id = 2").limit(10).to_pandas()
# --8<-- [end:scalar_index_filter]
assert len(result) == 1
def test_scalar_index_prefilter(tmp_db):
table = tmp_db.create_table(
"book_with_embeddings",
[
{"vector": [1.2, 1.3], "book_id": 1},
{"vector": [4.2, 4.3], "book_id": 2},
],
mode="overwrite",
)
db = tmp_db
# --8<-- [start:scalar_index_prefilter]
table = db.open_table("book_with_embeddings")
table.search([1.2] * 2).where("book_id != 3").limit(10).to_pandas()
# --8<-- [end:scalar_index_prefilter]
result = table.search([1.2] * 2).where("book_id != 3").limit(10).to_pandas()
assert len(result) == 2
def test_scalar_index_uuid(tmp_db):
# --8<-- [start:scalar_index_uuid_type]
import pyarrow as pa
# --8<-- [end:scalar_index_uuid_type]
# --8<-- [start:scalar_index_uuid_data]
def generate_random_names():
base_names = ["Alice", "Bob", "Carla", "David", "Eve", "Frank", "Grace"]
letter = random.choice(string.ascii_uppercase)
return f"{random.choice(base_names)} {letter}."
def generate_uuids(num_items):
return [uuid.uuid4().bytes for _ in range(num_items)]
# Generate some UUIDs and random names
n = 7
uuids = generate_uuids(n)
names = [generate_random_names() for _ in range(n)]
# --8<-- [end:scalar_index_uuid_data]
db = tmp_db
# --8<-- [start:scalar_index_uuid_table]
table_name = "index-on-uuid"
uuid_array = pa.array(uuids, pa.uuid())
name_array = pa.array(names, pa.string())
schema = pa.schema(
[
pa.field("id", pa.uuid()),
pa.field("name", pa.string()),
]
)
data_table = pa.Table.from_arrays([uuid_array, name_array], schema=schema)
table = db.create_table(table_name, data=data_table, mode="overwrite")
# --8<-- [end:scalar_index_uuid_table]
# --8<-- [start:scalar_index_uuid_wait]
index_name = "id_idx"
table.create_scalar_index("id")
table.wait_for_index([index_name])
# --8<-- [end:scalar_index_uuid_wait]
# --8<-- [start:scalar_index_uuid_upsert]
new_users = [
{"id": uuid.uuid4().bytes, "name": "Hannah D."},
{"id": uuid.uuid4().bytes, "name": "Ian B."},
]
# Insert or update using the UUID index
table.merge_insert(
"id"
).when_matched_update_all().when_not_matched_insert_all().execute(new_users)
# --8<-- [end:scalar_index_uuid_upsert]
assert table.list_indices()
result = table.search().limit(100).to_pandas()
assert len(result) == n + len(new_users)
@pytest.mark.asyncio
async def test_scalar_index_nested_fields(mem_db_async):
db = mem_db_async
# --8<-- [start:scalar_index_nested_fields]
import pyarrow as pa
from lancedb.index import BTree
metadata_type = pa.struct(
[
pa.field("user_id", pa.int32()),
pa.field("user.id", pa.int32()),
]
)
data = pa.Table.from_arrays(
[
pa.array([1, 2, 3], type=pa.int32()),
pa.array(
[
{"user_id": 10, "user.id": 100},
{"user_id": 20, "user.id": 200},
{"user_id": 30, "user.id": 300},
],
type=metadata_type,
),
],
names=["user_id", "metadata"],
)
table = await db.create_table("nested_scalar_index", data)
# Index a nested struct field.
await table.create_index(
"metadata.user_id", config=BTree(), name="nested_user_id_idx"
)
# Escape literal dots inside a segment with backticks.
await table.create_index(
"metadata.`user.id`", config=BTree(), name="escaped_user_id_idx"
)
# `columns` is returned as the canonical path you passed in.
for index in await table.list_indices():
print(index.name, index.columns)
# nested_user_id_idx ['metadata.user_id']
# escaped_user_id_idx ['metadata.`user.id`']
# --8<-- [end:scalar_index_nested_fields]
index_columns = {index.name: index.columns for index in await table.list_indices()}
assert index_columns["nested_user_id_idx"] == ["metadata.user_id"]
assert index_columns["escaped_user_id_idx"] == ["metadata.`user.id`"]
def test_fts_index_create(tmp_db):
table = tmp_db.create_table(
"fts-index-create",
[{"text": "hello world", "vector": [0.1, 0.2]}],
mode="overwrite",
)
db = tmp_db
# --8<-- [start:fts_index_create]
table_name = "fts-index-create"
table = db.open_table(table_name)
table.create_fts_index("text")
# --8<-- [end:fts_index_create]
assert table.list_indices()
def test_fts_index_wait(tmp_db):
table = tmp_db.create_table(
"fts-index-wait",
[{"text": "full text search"}],
mode="overwrite",
)
db = tmp_db
# --8<-- [start:fts_index_wait]
table_name = "fts-index-wait"
table = db.open_table(table_name)
table.create_fts_index("text")
index_name = "text_idx"
table.wait_for_index([index_name])
# --8<-- [end:fts_index_wait]
assert table.list_indices()
def test_fts_index_nested_field(tmp_db):
nested_schema = pa.struct([
pa.field("text", pa.string()),
pa.field("count", pa.int32()),
])
schema = pa.schema([
pa.field("id", pa.int64()),
pa.field("payload", nested_schema),
])
tmp_db.create_table(
"fts-index-nested",
pa.table(
{
"id": pa.array([1, 2], pa.int64()),
"payload": pa.array(
[
{"text": "Frodo was a happy puppy", "count": 1},
{"text": "puppy runs through the meadow", "count": 2},
],
type=nested_schema,
),
},
schema=schema,
),
mode="overwrite",
)
db = tmp_db
# --8<-- [start:fts_index_nested]
from lancedb.query import MatchQuery, PhraseQuery
table = db.open_table("fts-index-nested")
# Index a text leaf inside a struct column using a dotted path.
table.create_fts_index("payload.text", with_position=True)
# The same dotted path works in MatchQuery and PhraseQuery.
matches = (
table.search(MatchQuery("puppy", "payload.text")).limit(5).to_list()
)
phrases = (
table.search(PhraseQuery("puppy runs", "payload.text"))
.limit(5)
.to_list()
)
# --8<-- [end:fts_index_nested]
assert len(matches) > 0
assert all("puppy" in row["payload"]["text"] for row in matches)
assert len(phrases) > 0
assert all("puppy runs" in row["payload"]["text"] for row in phrases)
@pytest.mark.asyncio
async def test_fts_index_async(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
# --8<-- [start:fts_index_async]
import asyncio
import lancedb
import polars as pl
from lancedb.index import FTS
data = pl.DataFrame(
{
"id": [1, 2],
"text": [
"His first language is spanish",
"Her first language is english",
],
}
)
async def main(data: pl.DataFrame):
uri = "ex_lancedb"
db = await lancedb.connect_async(uri)
tbl = await db.create_table("my_text", data=data, mode="overwrite")
await tbl.create_index("text", config=FTS(language="English"))
response = await tbl.search("spanish", query_type="fts")
result = await response.limit(1).to_polars()
print(result)
return result
if __name__ == "__main__":
asyncio.run(main(data))
# --8<-- [end:fts_index_async]
result = await main(data)
assert result.height == 1
def test_gpu_index_snippets(tmp_db, monkeypatch):
table = tmp_db.create_table(
"gpu_index",
_make_vector_rows(32, 8),
mode="overwrite",
)
calls = []
def fake_create_index(*args, **kwargs):
calls.append(kwargs)
return None
monkeypatch.setattr(table, "create_index", fake_create_index)
# --8<-- [start:gpu_index_cuda]
table.create_index(
num_partitions=256,
num_sub_vectors=96,
accelerator="cuda",
)
# --8<-- [end:gpu_index_cuda]
# --8<-- [start:gpu_index_mps]
table.create_index(
num_partitions=256,
num_sub_vectors=96,
accelerator="mps",
)
# --8<-- [end:gpu_index_mps]
assert calls[0]["accelerator"] == "cuda"
assert calls[1]["accelerator"] == "mps"
def test_reindexing_incremental(tmp_db):
table = tmp_db.create_table(
"reindexing_incremental",
[{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}],
mode="overwrite",
)
db = tmp_db
# --8<-- [start:reindexing_incremental]
table = db.open_table("reindexing_incremental")
table.add([{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}])
table.optimize()
# --8<-- [end:reindexing_incremental]
result = table.search().limit(10).to_pandas()
assert len(result) == 2
docs/web-tests/py/test_integrations.py
+1767
View File
File diff suppressed because it is too large Load Diff
docs/web-tests/py/test_materialized_view.py
+69
View File
@@ -0,0 +1,69 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""Examples for docs/web/feature-engineering/materialized-views.mdx."""
def test_materialized_view_create_and_refresh(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
# --8<-- [start:materialized_view_create]
import lancedb
db = lancedb.connect("./.lancedb")
table = db.create_table(
"people", [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}]
)
view = db.create_materialized_view(
"adults",
"people",
select=["name", ("shout", "upper(name)")],
where="age >= 18",
)
print(view.table.count_rows())
# 1
# --8<-- [end:materialized_view_create]
assert view.table.count_rows() == 1
# --8<-- [start:materialized_view_refresh]
table.add([{"name": "bea", "age": 41}])
result = view.refresh()
print(result.rows_written)
# 2
# --8<-- [end:materialized_view_refresh]
assert result.mode == "rebuild"
assert result.rows_written == 2
assert sorted(row["shout"] for row in view.table.to_arrow().to_pylist()) == [
"ADA",
"BEA",
]
def test_materialized_view_with_no_data(tmp_path):
import lancedb
db = lancedb.connect(tmp_path)
db.create_table("people", [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}])
view = db.create_materialized_view(
"adults", "people", where="age >= 18", with_no_data=True
)
assert view.table.count_rows() == 0
assert view.refresh().rows_written == 1
assert view.table.count_rows() == 1
def test_materialized_view_refresh_to_source_version(tmp_path):
import lancedb
db = lancedb.connect(tmp_path)
table = db.create_table("people", [{"name": "ada", "age": 36}])
view = db.create_materialized_view("adults", "people", where="age >= 18")
first = table.version
table.add([{"name": "bea", "age": 41}])
assert view.refresh().rows_written == 2
result = view.refresh(source_version=first)
assert (result.source_version, result.rows_written) == (first, 1)
assert view.table.count_rows() == 1
docs/web-tests/py/test_multimodal.py
+232
View File
@@ -0,0 +1,232 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import pytest
try:
import lancedb
import numpy as np
import pyarrow as pa
import io
from PIL import Image
except ImportError:
pass
# --8<-- [start:multimodal_imports]
import lancedb
import pyarrow as pa
import pandas as pd
import numpy as np
import io
from PIL import Image
# --8<-- [end:multimodal_imports]
def test_multimodal_ingestion(db_path_factory):
# Ensure dependencies are available
pytest.importorskip("PIL")
pytest.importorskip("lancedb")
pytest.importorskip("numpy")
# --8<-- [start:create_dummy_data]
# Create some dummy images
def create_dummy_image(color):
img = Image.new('RGB', (100, 100), color=color)
buf = io.BytesIO()
img.save(buf, format='PNG')
return buf.getvalue()
# Create dataset with metadata, vectors, and image blobs
data = [
{
"id": 1,
"filename": "red_square.png",
"vector": np.random.rand(128).astype(np.float32),
"image_blob": create_dummy_image('red'),
"label": "red"
},
{
"id": 2,
"filename": "blue_square.png",
"vector": np.random.rand(128).astype(np.float32),
"image_blob": create_dummy_image('blue'),
"label": "blue"
}
]
# --8<-- [end:create_dummy_data]
# --8<-- [start:define_schema]
# Define schema explicitly to ensure image_blob is treated as binary
schema = pa.schema([
pa.field("id", pa.int32()),
pa.field("filename", pa.string()),
pa.field("vector", pa.list_(pa.float32(), 128)),
pa.field("image_blob", pa.binary()), # Important: Use pa.binary() for blobs
pa.field("label", pa.string())
])
# --8<-- [end:define_schema]
db_uri = db_path_factory("multimodal_db")
db = lancedb.connect(db_uri)
# --8<-- [start:ingest_data]
tbl = db.create_table("images", data=data, schema=schema, mode="overwrite")
# --8<-- [end:ingest_data]
assert len(tbl) == 2
# --8<-- [start:search_data]
# Search for similar images
query_vector = np.random.rand(128).astype(np.float32)
results = tbl.search(query_vector).limit(1).to_pandas()
# --8<-- [end:search_data]
# --8<-- [start:process_results]
# Convert back to PIL Image
for _, row in results.iterrows():
image_bytes = row['image_blob']
image = Image.open(io.BytesIO(image_bytes))
print(f"Retrieved image: {row['filename']}, Size: {image.size}")
# You can now use 'image' with other libraries or display it
# --8<-- [end:process_results]
assert len(results) == 1
def test_blob_api_definition(db_path_factory):
# --8<-- [start:blob_api_schema]
import pyarrow as pa
# Define schema with Blob API metadata for lazy loading
schema = pa.schema([
pa.field("id", pa.int64()),
pa.field(
"video",
pa.large_binary(),
metadata={"lance-encoding:blob": "true"} # Enable Blob API
),
])
# --8<-- [end:blob_api_schema]
# --8<-- [start:blob_api_ingest]
import lancedb
db = lancedb.connect(db_path_factory("blob_db"))
# Create sample data
data = [
{"id": 1, "video": b"fake_video_bytes_1"},
{"id": 2, "video": b"fake_video_bytes_2"}
]
# Create the table
tbl = db.create_table("videos", data=data, schema=schema)
# --8<-- [end:blob_api_ingest]
assert len(tbl) == 2
def test_blob_api_to_pandas(db_path_factory):
db = lancedb.connect(db_path_factory("blob_to_pandas_db"))
schema = pa.schema([
pa.field("id", pa.int64()),
pa.field(
"video",
pa.large_binary(),
metadata={"lance-encoding:blob": "true"},
),
])
tbl = db.create_table(
"videos",
data=[
{"id": 1, "video": b"fake_video_bytes_1"},
{"id": 2, "video": b"fake_video_bytes_2"},
],
schema=schema,
mode="overwrite",
)
# --8<-- [start:blob_api_to_pandas]
# Default: blob columns come back lazily
df_lazy = tbl.to_pandas()
# Materialize blob bytes eagerly
df_bytes = tbl.to_pandas(blob_mode="bytes")
# Return descriptors instead of payloads
df_desc = tbl.to_pandas(blob_mode="descriptions")
# Forward extra kwargs to PyArrow's to_pandas
df_typed = tbl.to_pandas(split_blocks=True, self_destruct=True)
# --8<-- [end:blob_api_to_pandas]
assert len(df_lazy) == 2
assert isinstance(df_bytes["video"].iloc[0], bytes)
assert df_bytes["video"].tolist() == [
b"fake_video_bytes_1",
b"fake_video_bytes_2",
]
assert len(df_desc) == 2
assert len(df_typed) == 2
@pytest.mark.asyncio
async def test_query_to_pandas_kwargs(db_path_factory):
schema = pa.schema([
pa.field("id", pa.int64()),
pa.field("vector", pa.list_(pa.float32(), 128)),
pa.field(
"video",
pa.large_binary(),
metadata={"lance-encoding:blob": "true"},
),
])
data = [
{
"id": i,
"vector": np.random.rand(128).astype(np.float32),
"video": f"fake_video_bytes_{i}".encode(),
}
for i in range(10)
]
db = lancedb.connect(db_path_factory("query_to_pandas_db"))
tbl = db.create_table("search_demo", data=data, schema=schema, mode="overwrite")
async_db = await lancedb.connect_async(
str(db_path_factory("query_to_pandas_async_db"))
)
tbl_async = await async_db.create_table(
"search_demo", data=data, schema=schema, mode="overwrite"
)
query_vector = np.random.rand(128).astype(np.float32)
# --8<-- [start:query_to_pandas_kwargs]
# Plain scan query: blob_mode is supported end to end
df_lazy = (
tbl.search()
.where("id = 1")
.select(["id", "video"])
.to_pandas(blob_mode="lazy")
)
# Same call shape works on async query builders
df_bytes = await (
tbl_async.query()
.where("id = 1")
.select(["id", "video"])
.to_pandas(blob_mode="bytes")
)
# Vector / FTS / hybrid queries can't materialize blob columns,
# so omit them from the projection
df_vec = (
tbl.search(query_vector)
.limit(10)
.select(["id", "vector"])
.to_pandas(split_blocks=True, self_destruct=True)
)
# --8<-- [end:query_to_pandas_kwargs]
assert len(df_lazy) == 1
assert len(df_bytes) == 1
assert df_bytes["video"].iloc[0] == b"fake_video_bytes_1"
assert len(df_vec) == 10
assert "video" not in df_vec.columns
docs/web-tests/py/test_quickstart.py
+240
View File
@@ -0,0 +1,240 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import lancedb
import pytest
def test_quickstart(db_path_factory):
uri = db_path_factory("quickstart_db")
db = lancedb.connect(uri)
# --8<-- [start:quickstart_data]
data = [
{
"id": "1",
"name": "King Arthur",
"role": "King",
"description": "Leader of Camelot and wielder of Excalibur.",
"stats": {"strength": 4, "magic": 1, "leadership": 5, "wisdom": 4},
"vector": [0.7, 0.1, 0.9, 0.7],
},
{
"id": "2",
"name": "Merlin",
"role": "Wizard",
"description": "Advisor and prophet with deep magical knowledge.",
"stats": {"strength": 2, "magic": 5, "leadership": 4, "wisdom": 5},
"vector": [0.2, 0.9, 0.4, 0.9],
},
{
"id": "3",
"name": "Sir Lancelot",
"role": "Knight",
"description": "Legendary knight known for courage and combat skill.",
"stats": {"strength": 5, "magic": 1, "leadership": 3, "wisdom": 3},
"vector": [0.9, 0.1, 0.5, 0.4],
},
]
# --8<-- [end:quickstart_data]
# --8<-- [start:quickstart_create_table]
table = db.create_table("characters", data=data, mode="overwrite")
# --8<-- [end:quickstart_create_table]
assert len(table) == 3
# Drop the table to test create without overwrite
db.drop_table("characters")
# --8<-- [start:quickstart_create_table_no_overwrite]
table = db.create_table("characters", data=data)
# --8<-- [end:quickstart_create_table_no_overwrite]
assert len(table) == 3
# --8<-- [start:quickstart_vector_search_1]
# Search for examples similar to a "wise magical advisor"
query_vector = [0.2, 0.8, 0.4, 0.9]
# Ensure you run `pip install polars` beforehand
result = (
table.search(query_vector)
.select(["name", "role", "description", "_distance"])
.limit(2)
.to_polars()
)
print(result)
# --8<-- [end:quickstart_vector_search_1]
assert result.head(1)["name"][0] == "Merlin"
# --8<-- [start:quickstart_curate_with_metadata]
curated = (
table.search(query_vector)
.where("stats.magic >= 4")
.select(["name", "role", "description", "_distance"])
.limit(2)
.to_polars()
)
print(curated)
# --8<-- [end:quickstart_curate_with_metadata]
assert curated.head(1)["name"][0] == "Merlin"
# --8<-- [start:quickstart_output_pandas]
# Ensure you run `pip install pandas` beforehand
result = table.search(query_vector).limit(2).to_pandas()
print(result)
# --8<-- [end:quickstart_output_pandas]
assert result.iloc[0]["name"] == "Merlin"
# --8<-- [start:quickstart_add_feature]
table.add_columns(
{
"power_score": "cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)"
}
)
# --8<-- [end:quickstart_add_feature]
assert "power_score" in table.schema.names
# --8<-- [start:quickstart_query_feature]
features = table.search().select(["name", "role", "power_score"]).to_polars()
print(features)
# --8<-- [end:quickstart_query_feature]
assert "power_score" in features.columns
# --8<-- [start:quickstart_multimodal_bytes]
from pathlib import Path
image_path = Path("docs/web/static/assets/images/quickstart/sir-lancelot.jpg")
image_bytes = image_path.read_bytes()
multimodal_table = db.create_table(
"character_images",
data=[
{
"id": "lancelot",
"description": "Portrait of Sir Lancelot",
"image": image_bytes,
"vector": [0.9, 0.1, 0.5, 0.4],
}
],
mode="overwrite",
)
# --8<-- [end:quickstart_multimodal_bytes]
assert len(multimodal_table) == 1
# --8<-- [start:quickstart_open_table]
table = db.open_table("characters")
# --8<-- [end:quickstart_open_table]
# --8<-- [start:quickstart_add_data]
more_data = [
{
"id": "4",
"name": "Morgana",
"role": "Sorceress",
"description": "Powerful sorceress of Avalon.",
"stats": {"strength": 2, "magic": 5, "leadership": 4, "wisdom": 4},
"vector": [0.3, 0.9, 0.6, 0.8],
"power_score": 3.75,
},
]
# Add data to table
table.add(more_data)
# --8<-- [end:quickstart_add_data]
assert len(table) == 4
# --8<-- [start:quickstart_vector_search_2]
# Search for examples similar to a "powerful sorceress"
query_vector = [0.3, 0.9, 0.6, 0.8]
results = table.search(query_vector).limit(2).to_polars()
print(results)
# --8<-- [end:quickstart_vector_search_2]
assert results.head(1)["name"][0] == "Morgana"
# --8<-- [start:quickstart_computed_feature]
table.add_columns(computed={"combat_score": "(stats.magic + stats.strength) / 2.0"})
table.refresh_column("combat_score")
# --8<-- [end:quickstart_computed_feature]
scores = (
table.search().select(["combat_score"]).to_arrow()["combat_score"].to_pylist()
)
assert scores == [2.5, 3.5, 3.0, 3.5]
# --8<-- [start:quickstart_training]
import torch
from lancedb.permutation import Permutation
dataset = Permutation.identity(table).select_columns(
["power_score", "combat_score"]
)
dataloader = torch.utils.data.DataLoader(dataset, batch_size=1024, shuffle=True)
# --8<-- [end:quickstart_training]
batches = list(dataloader)
assert len(batches) == 1
assert set(batches[0]) == {"power_score", "combat_score"}
assert all(isinstance(value, torch.Tensor) for value in batches[0].values())
assert sorted(batches[0]["combat_score"].tolist()) == sorted(scores)
# --8<-- [start:quickstart_training_projection]
dataset = Permutation.identity(table).select_columns(["power_score"])
dataloader = torch.utils.data.DataLoader(dataset, batch_size=1024)
# --8<-- [end:quickstart_training_projection]
batch = next(iter(dataloader))
assert set(batch) == {"power_score"}
assert batch["power_score"].tolist() == [3.5, 4.0, 3.0, 3.75]
@pytest.mark.asyncio
async def test_quickstart_async_api(db_path_factory):
db_uri = db_path_factory("quickstart_async_db")
import lancedb
async_db = await lancedb.connect_async(db_uri)
# --8<-- [start:quickstart_data_async]
data = [
{
"id": "1",
"name": "King Arthur",
"role": "King",
"description": "Leader of Camelot and wielder of Excalibur.",
"stats": {"strength": 4, "magic": 1, "leadership": 5, "wisdom": 4},
"vector": [0.7, 0.1, 0.9, 0.7],
},
{
"id": "2",
"name": "Merlin",
"role": "Wizard",
"description": "Advisor and prophet with deep magical knowledge.",
"stats": {"strength": 2, "magic": 5, "leadership": 4, "wisdom": 5},
"vector": [0.2, 0.9, 0.4, 0.9],
},
{
"id": "3",
"name": "Sir Lancelot",
"role": "Knight",
"description": "Legendary knight known for courage and combat skill.",
"stats": {"strength": 5, "magic": 1, "leadership": 3, "wisdom": 3},
"vector": [0.9, 0.1, 0.5, 0.4],
},
]
# --8<-- [end:quickstart_data_async]
# --8<-- [start:quickstart_create_table_async]
async_table = await async_db.create_table(
"characters",
data=data,
mode="overwrite",
)
# --8<-- [end:quickstart_create_table_async]
# --8<-- [start:quickstart_vector_search_1_async]
# Search for examples similar to a "wise magical advisor"
query_vector = [0.2, 0.8, 0.4, 0.9]
# Ensure you run `pip install polars` beforehand
async_result = await (
await async_table.search(query_vector)
).select(["name", "role", "description", "_distance"]).limit(2).to_polars()
print(async_result)
# --8<-- [end:quickstart_vector_search_1_async]
assert async_result.head(1)["name"][0] == "Merlin"
docs/web-tests/py/test_search.py
+730
View File
@@ -0,0 +1,730 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
# --8<-- [start:import-lancedb]
import lancedb
# --8<-- [end:import-lancedb]
# --8<-- [start:import-numpy]
from lancedb.query import BoostQuery, MatchQuery
import numpy as np
import pyarrow as pa
# --8<-- [end:import-numpy]
# --8<-- [start:import-datetime]
from datetime import datetime
# --8<-- [end:import-datetime]
# --8<-- [start:import-lancedb-pydantic]
from lancedb.pydantic import Vector, LanceModel
# --8<-- [end:import-lancedb-pydantic]
# --8<-- [start:import-pydantic-base-model]
from pydantic import BaseModel
# --8<-- [end:import-pydantic-base-model]
# --8<-- [start:import-lancedb-fts]
from lancedb.index import FTS
# --8<-- [end:import-lancedb-fts]
# --8<-- [start:import-os]
import os
# --8<-- [end:import-os]
# --8<-- [start:import-embeddings]
from lancedb.embeddings import get_registry
# --8<-- [end:import-embeddings]
import pytest
# --8<-- [start:class-definition]
class Metadata(BaseModel):
source: str
timestamp: datetime
class Document(BaseModel):
content: str
meta: Metadata
class LanceSchema(LanceModel):
id: str
vector: Vector(1536)
payload: Document
# --8<-- [end:class-definition]
def test_vector_search():
# --8<-- [start:exhaustive_search]
uri = "data/sample-lancedb"
db = lancedb.connect(uri)
data = [
{"vector": row, "item": f"item {i}"}
for i, row in enumerate(np.random.random((10_000, 1536)).astype("float32"))
]
tbl = db.create_table("vector_search", data=data, mode="overwrite")
tbl.search(np.random.random((1536))).limit(10).to_list()
# --8<-- [end:exhaustive_search]
# --8<-- [start:exhaustive_search_cosine]
tbl.search(np.random.random((1536))).distance_type("cosine").limit(10).to_list()
# --8<-- [end:exhaustive_search_cosine]
# --8<-- [start:create_table_with_nested_schema]
# Let's add 100 sample rows to our dataset
data = [
LanceSchema(
id=f"id{i}",
vector=np.random.randn(1536),
payload=Document(
content=f"document{i}",
meta=Metadata(source=f"source{i % 10}", timestamp=datetime.now()),
),
)
for i in range(100)
]
# Synchronous client
tbl = db.create_table("documents", data=data, mode="overwrite")
# --8<-- [end:create_table_with_nested_schema]
# --8<-- [start:search_result_as_pyarrow]
tbl.search(np.random.randn(1536)).to_arrow()
# --8<-- [end:search_result_as_pyarrow]
# --8<-- [start:search_result_as_pandas]
tbl.search(np.random.randn(1536)).to_pandas()
# --8<-- [end:search_result_as_pandas]
# --8<-- [start:search_result_as_pandas_flatten_true]
tbl.search(np.random.randn(1536)).to_pandas(flatten=True)
# --8<-- [end:search_result_as_pandas_flatten_true]
# --8<-- [start:search_result_as_pandas_flatten_1]
tbl.search(np.random.randn(1536)).to_pandas(flatten=1)
# --8<-- [end:search_result_as_pandas_flatten_1]
# --8<-- [start:search_result_as_list]
tbl.search(np.random.randn(1536)).to_list()
# --8<-- [end:search_result_as_list]
# --8<-- [start:search_result_as_pydantic]
tbl.search(np.random.randn(1536)).to_pydantic(LanceSchema)
# --8<-- [end:search_result_as_pydantic]
@pytest.mark.asyncio
async def test_vector_search_async():
# --8<-- [start:exhaustive_search_async]
uri = "data/sample-lancedb"
async_db = await lancedb.connect_async(uri)
data = [
{"vector": row, "item": f"item {i}"}
for i, row in enumerate(np.random.random((10_000, 1536)).astype("float32"))
]
async_tbl = await async_db.create_table(
"vector_search_async", data=data, mode="overwrite"
)
(await (await async_tbl.search(np.random.random((1536)))).limit(10).to_list())
# --8<-- [end:exhaustive_search_async]
# --8<-- [start:exhaustive_search_async_cosine]
(
await (await async_tbl.search(np.random.random((1536))))
.distance_type("cosine")
.limit(10)
.to_list()
)
# --8<-- [end:exhaustive_search_async_cosine]
# --8<-- [start:create_table_async_with_nested_schema]
# Let's add 100 sample rows to our dataset
data = [
LanceSchema(
id=f"id{i}",
vector=np.random.randn(1536),
payload=Document(
content=f"document{i}",
meta=Metadata(source=f"source{i % 10}", timestamp=datetime.now()),
),
)
for i in range(100)
]
async_tbl = await async_db.create_table(
"documents_async", data=data, mode="overwrite"
)
# --8<-- [end:create_table_async_with_nested_schema]
# --8<-- [start:search_result_async_as_pyarrow]
await (await async_tbl.search(np.random.randn(1536))).to_arrow()
# --8<-- [end:search_result_async_as_pyarrow]
# --8<-- [start:search_result_async_as_pandas]
await (await async_tbl.search(np.random.randn(1536))).to_pandas()
# --8<-- [end:search_result_async_as_pandas]
# --8<-- [start:search_result_async_as_list]
await (await async_tbl.search(np.random.randn(1536))).to_list()
# --8<-- [end:search_result_async_as_list]
def test_fts_fuzzy_query():
uri = "data/fuzzy-example"
db = lancedb.connect(uri)
table = db.create_table(
"my_table_fts_fuzzy",
data=pa.table(
{
"text": [
"fa",
"fo", # spellchecker:disable-line
"fob",
"focus",
"foo",
"food",
"foul",
]
}
),
mode="overwrite",
)
table.create_fts_index("text", replace=True)
results = table.search(MatchQuery("foo", "text", fuzziness=1)).to_pandas()
assert len(results) == 4
assert set(results["text"].to_list()) == {
"foo",
"fo", # 1 deletion # spellchecker:disable-line
"fob", # 1 substitution
"food", # 1 insertion
}
def test_fts_boost_query():
uri = "data/boost-example"
db = lancedb.connect(uri)
table = db.create_table(
"my_table_fts_boost",
data=pa.table(
{
"title": [
"The Hidden Gems of Travel",
"Exploring Nature's Wonders",
"Cultural Treasures Unveiled",
"The Nightlife Chronicles",
"Scenic Escapes and Challenges",
],
"desc": [
"A vibrant city with occasional traffic jams.",
"Beautiful landscapes but overpriced tourist spots.",
"Rich cultural heritage but humid summers.",
"Bustling nightlife but noisy streets.",
"Scenic views but limited public transport options.",
],
}
),
mode="overwrite",
)
table.create_fts_index("desc", replace=True)
results = table.search(
BoostQuery(
MatchQuery("beautiful, cultural, nightlife", "desc"),
MatchQuery("bad traffic jams, overpriced", "desc"),
),
).to_pandas()
# we will hit 3 results because the positive query has 3 hits
assert len(results) == 3
# the one containing "overpriced" will be negatively boosted,
# so it will be the last one
assert (
results["desc"].to_list()[2]
== "Beautiful landscapes but overpriced tourist spots."
)
def test_fts_native():
# --8<-- [start:basic_fts]
uri = "data/sample-lancedb"
db = lancedb.connect(uri)
table = db.create_table(
"my_table_fts",
data=[
{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"},
{"vector": [5.9, 26.5], "text": "There are several kittens playing"},
],
mode="overwrite",
)
table.create_fts_index("text")
table.search("puppy").limit(10).select(["text"]).to_list()
# [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}]
# ...
# --8<-- [end:basic_fts]
# --8<-- [start:fts_config_stem]
table.create_fts_index("text", tokenizer_name="en_stem", replace=True)
# --8<-- [end:fts_config_stem]
# --8<-- [start:fts_config_folding]
table.create_fts_index(
"text",
language="French",
stem=True,
ascii_folding=True,
replace=True,
)
# --8<-- [end:fts_config_folding]
# --8<-- [start:fts_prefiltering]
table.search("puppy").limit(10).where("text='foo'", prefilter=True).to_list()
# --8<-- [end:fts_prefiltering]
# --8<-- [start:fts_postfiltering]
table.search("puppy").limit(10).where("text='foo'", prefilter=False).to_list()
# --8<-- [end:fts_postfiltering]
# --8<-- [start:fts_with_position]
table.create_fts_index("text", with_position=True, replace=True)
# --8<-- [end:fts_with_position]
# --8<-- [start:fts_incremental_index]
table.add([{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}])
table.optimize()
# --8<-- [end:fts_incremental_index]
@pytest.mark.asyncio
async def test_fts_native_async():
# --8<-- [start:basic_fts_async]
uri = "data/sample-lancedb"
async_db = await lancedb.connect_async(uri)
async_tbl = await async_db.create_table(
"my_table_fts_async",
data=[
{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"},
{"vector": [5.9, 26.5], "text": "There are several kittens playing"},
],
mode="overwrite",
)
# async API uses our native FTS algorithm
await async_tbl.create_index("text", config=FTS())
await (await async_tbl.search("puppy")).select(["text"]).limit(10).to_list()
# [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}]
# ...
# --8<-- [end:basic_fts_async]
# --8<-- [start:fts_config_stem_async]
await async_tbl.create_index(
"text", config=FTS(language="English", stem=True, remove_stop_words=True)
)
# --8<-- [end:fts_config_stem_async]
# --8<-- [start:fts_config_folding_async]
await async_tbl.create_index(
"text", config=FTS(language="French", stem=True, ascii_folding=True)
)
# --8<-- [end:fts_config_folding_async]
# --8<-- [start:fts_prefiltering_async]
await (await async_tbl.search("puppy")).limit(10).where("text='foo'").to_list()
# --8<-- [end:fts_prefiltering_async]
# --8<-- [start:fts_postfiltering_async]
await (
(await async_tbl.search("puppy"))
.limit(10)
.where("text='foo'")
.postfilter()
.to_list()
)
# --8<-- [end:fts_postfiltering_async]
# --8<-- [start:fts_with_position_async]
await async_tbl.create_index("text", config=FTS(with_position=True))
# --8<-- [end:fts_with_position_async]
# --8<-- [start:fts_incremental_index_async]
await async_tbl.add([{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}])
await async_tbl.optimize()
# --8<-- [end:fts_incremental_index_async]
def _vectors(n, dim, seed=0, column="vector"):
rng = np.random.default_rng(seed)
return [
{column: rng.random(dim).astype("float32"), "id": i} for i in range(n)
]
def _build_indexed_table(db, name, dim=128, n=512):
tbl = db.create_table(name, _vectors(n, dim), mode="overwrite")
tbl.create_index(metric="cosine", num_partitions=4, num_sub_vectors=8)
return tbl
def test_vs_distance_metric_and_brute_force(tmp_db):
tbl = tmp_db.create_table("vs_metric", _vectors(64, 1536), mode="overwrite")
# --8<-- [start:configure_distance_metric]
tbl.search(np.random.random((1536))).distance_type("cosine").limit(10).to_list()
# --8<-- [end:configure_distance_metric]
# --8<-- [start:brute_force_search]
tbl.search(np.random.random((1536))).limit(3).to_list()
# --8<-- [end:brute_force_search]
def test_vs_select_vector_column(tmp_db):
db = tmp_db
# --8<-- [start:select_vector_column]
import pyarrow as pa
schema = pa.schema([
pa.field("id", pa.int32()),
pa.field(
"image",
pa.struct([pa.field("embedding", pa.list_(pa.float32(), 2))]),
),
])
table = db.create_table(
"nested",
data=[{"id": 0, "image": {"embedding": [0.0, 1.0]}}],
schema=schema,
)
# Inferred: the only vector leaf is `image.embedding`.
table.search([0.0, 1.0]).limit(1).to_list()
# Explicit: required when more than one vector column matches.
table.search([0.0, 1.0], vector_column_name="image.embedding").limit(1).to_list()
# --8<-- [end:select_vector_column]
def test_vs_index_nested_column(tmp_db):
dim = 16
schema = pa.schema(
[
pa.field("id", pa.int32()),
pa.field(
"image",
pa.struct([pa.field("embedding", pa.list_(pa.float32(), dim))]),
),
]
)
rng = np.random.default_rng(0)
data = [
{"id": i, "image": {"embedding": rng.random(dim).astype("float32").tolist()}}
for i in range(512)
]
table = tmp_db.create_table(
"nested_index", data=data, schema=schema, mode="overwrite"
)
# --8<-- [start:index_nested_column]
table.create_index(vector_column_name="image.embedding")
# --8<-- [end:index_nested_column]
assert table.list_indices()
def test_vs_indexed_queries(tmp_db):
table = _build_indexed_table(tmp_db, "vs_indexed", dim=128, n=512)
embedding = np.random.random(128)
# --8<-- [start:exact_vs_approximate_distances]
# Indexed ANN search without refinement (fast, approximate `_distance`)
fast_results = (
table.search(embedding)
.limit(10)
.to_pandas()
)
# Recompute distances on full vectors for reranked candidates
exact_distance_results = (
table.search(embedding)
.limit(10)
.refine_factor(1)
.to_pandas()
)
# Rerank a larger candidate set for better recall (higher latency)
higher_recall_results = (
table.search(embedding)
.limit(10)
.refine_factor(20)
.to_pandas()
)
# --8<-- [end:exact_vs_approximate_distances]
# --8<-- [start:bypass_vector_index]
table.search(embedding).bypass_vector_index().limit(5).to_pandas()
# --8<-- [end:bypass_vector_index]
assert len(fast_results) == 10
assert len(exact_distance_results) == 10
assert len(higher_recall_results) == 10
def test_vs_fast_search(tmp_db, monkeypatch):
table = _build_indexed_table(tmp_db, "vs_fast", dim=128, n=512)
embedding = np.random.random(128)
# `fast_search` is an Enterprise/async query flag; strip it so the snippet
# runs against a local OSS table without changing what readers see.
real_search = table.search
monkeypatch.setattr(
table,
"search",
lambda *a, **k: real_search(
*a, **{key: val for key, val in k.items() if key != "fast_search"}
),
)
# --8<-- [start:fast_search]
table.search(embedding, fast_search=True).limit(5).to_pandas()
# --8<-- [end:fast_search]
def test_vs_distance_range(tmp_db):
tbl = tmp_db.create_table("vs_distance_range", _vectors(256, 256), mode="overwrite")
# --8<-- [start:search_distance_range]
query = np.random.random(256)
# Search for the vectors within the range of [0.1, 0.5)
tbl.search(query).distance_range(0.1, 0.5).to_arrow()
# Search for the vectors with the distance less than 0.5
tbl.search(query).distance_range(upper_bound=0.5).to_arrow()
# Search for the vectors with the distance greater or equal to 0.1
tbl.search(query).distance_range(lower_bound=0.1).to_arrow()
# --8<-- [end:search_distance_range]
def test_vs_multivector_search(tmp_db):
schema = pa.schema(
[
pa.field("id", pa.int64()),
pa.field("vector", pa.list_(pa.list_(pa.float32(), 256))),
]
)
rng = np.random.default_rng(0)
data = [{"id": i, "vector": rng.random(size=(2, 256)).tolist()} for i in range(64)]
tbl = tmp_db.create_table(
"vs_multivector", data=data, schema=schema, mode="overwrite"
)
# --8<-- [start:multivector_search]
query_multi = np.random.random(size=(2, 256))
results_multi = tbl.search(query_multi).limit(5).to_pandas()
# --8<-- [end:multivector_search]
assert len(results_multi) <= 5
def test_vs_binary_search(tmp_db):
db = tmp_db
# --8<-- [start:search_binary_vectors]
import numpy as np
import pyarrow as pa
schema = pa.schema(
[
pa.field("id", pa.int64()),
# for dim=256, lance stores every 8 bits in a byte
# so the vector field should be a list of 256 / 8 = 32 bytes
pa.field("vector", pa.list_(pa.uint8(), 32)),
]
)
tbl = db.create_table("my_binary_vectors", schema=schema)
data = []
for i in range(1024):
vector = np.random.randint(0, 2, size=256)
# pack the binary vector into bytes to save space
packed_vector = np.packbits(vector)
data.append(
{
"id": i,
"vector": packed_vector,
}
)
tbl.add(data)
query = np.random.randint(0, 2, size=256)
packed_query = np.packbits(query)
tbl.search(packed_query).distance_type("hamming").to_arrow()
# --8<-- [end:search_binary_vectors]
def test_vs_enterprise_filtering(tmp_db, monkeypatch):
import sys
import types
dim = 384
rng = np.random.default_rng(0)
rows = [
{
"vector": rng.random(dim).astype("float32"),
"text": f"story {i}",
"keywords": f"kw{i}",
"label": i % 4,
}
for i in range(50)
]
tmp_db.create_table("lancedb-enterprise-quickstart", data=rows, mode="overwrite")
db = tmp_db
# Mock the Hugging Face dataset loader to avoid network downloads.
class _FakeDataset:
def __init__(self, n):
self._emb = [rng.random(dim).astype("float32") for _ in range(n)]
self._kw = [f"kw{i}" for i in range(n)]
def __getitem__(self, key):
if isinstance(key, str):
return {"keywords_embeddings": self._emb, "keywords": self._kw}[key]
return {"keywords": self._kw[key], "keywords_embeddings": self._emb[key]}
fake_datasets = types.ModuleType("datasets")
fake_datasets.load_dataset = lambda *a, **k: _FakeDataset(10)
monkeypatch.setitem(sys.modules, "datasets", fake_datasets)
# --8<-- [start:vector_search_prefilter]
from datasets import load_dataset
# Load query vector from dataset
query_dataset = load_dataset("sunhaozhepy/ag_news_sbert_keywords_embeddings", split="test[5000:5001]")
print(f"Query keywords: {query_dataset[0]['keywords']}")
query_embed = query_dataset["keywords_embeddings"][0]
# Open table and perform search
table_name = "lancedb-enterprise-quickstart"
table = db.open_table(table_name)
# Vector search with filters (pre-filtering is the default)
search_results = (
table.search(query_embed)
.where("label > 2")
.select(["text", "keywords", "label"])
.limit(5)
.to_pandas()
)
print("Search results (with pre-filtering):")
print(search_results)
# --8<-- [end:vector_search_prefilter]
# --8<-- [start:vector_search_postfilter]
results_post_filtered = (
table.search(query_embed)
.where("label > 1", prefilter=False)
.select(["text", "keywords", "label"])
.limit(5)
.to_pandas()
)
print("Vector search results with post-filter:")
print(results_post_filtered)
# --8<-- [end:vector_search_postfilter]
# --8<-- [start:batch_search]
# Load a batch of query embeddings
query_dataset = load_dataset(
"sunhaozhepy/ag_news_sbert_keywords_embeddings", split="test[5000:5005]"
)
query_embeds = query_dataset["keywords_embeddings"]
batch_results = table.search(query_embeds).limit(5).to_pandas()
print(batch_results)
# --8<-- [end:batch_search]
@pytest.mark.skip()
def test_hybrid_search():
# --8<-- [start:import-openai]
import openai
# --8<-- [end:import-openai]
# --8<-- [start:openai-embeddings]
# Ingest embedding function in LanceDB table
# Configuring the environment variable OPENAI_API_KEY
if "OPENAI_API_KEY" not in os.environ:
# OR set the key here as a variable
openai.api_key = "sk-..."
embeddings = get_registry().get("openai").create()
# --8<-- [end:openai-embeddings]
# --8<-- [start:class-Documents]
class Documents(LanceModel):
vector: Vector(embeddings.ndims()) = embeddings.VectorField()
text: str = embeddings.SourceField()
# --8<-- [end:class-Documents]
# --8<-- [start:basic_hybrid_search]
data = [
{"text": "rebel spaceships striking from a hidden base"},
{"text": "have won their first victory against the evil Galactic Empire"},
{"text": "during the battle rebel spies managed to steal secret plans"},
{"text": "to the Empire's ultimate weapon the Death Star"},
]
uri = "data/sample-lancedb"
db = lancedb.connect(uri)
table = db.create_table("documents", schema=Documents)
# ingest docs with auto-vectorization
table.add(data)
# Create a fts index before the hybrid search
table.create_fts_index("text")
# hybrid search with default re-ranker
table.search("flower moon", query_type="hybrid").to_pandas()
# --8<-- [end:basic_hybrid_search]
# --8<-- [start:hybrid_search_pass_vector_text]
vector_query = [0.1, 0.2, 0.3, 0.4, 0.5]
text_query = "flower moon"
(
table.search(query_type="hybrid")
.vector(vector_query)
.text(text_query)
.limit(5)
.to_pandas()
)
# --8<-- [end:hybrid_search_pass_vector_text]
@pytest.mark.skip
async def test_hybrid_search_async():
import openai
# --8<-- [start:openai-embeddings]
# Ingest embedding function in LanceDB table
# Configuring the environment variable OPENAI_API_KEY
if "OPENAI_API_KEY" not in os.environ:
# OR set the key here as a variable
openai.api_key = "sk-..."
embeddings = get_registry().get("openai").create()
# --8<-- [end:openai-embeddings]
# --8<-- [start:class-Documents]
class Documents(LanceModel):
vector: Vector(embeddings.ndims()) = embeddings.VectorField()
text: str = embeddings.SourceField()
# --8<-- [end:class-Documents]
# --8<-- [start:basic_hybrid_search_async]
uri = "data/sample-lancedb"
async_db = await lancedb.connect_async(uri)
data = [
{"text": "rebel spaceships striking from a hidden base"},
{"text": "have won their first victory against the evil Galactic Empire"},
{"text": "during the battle rebel spies managed to steal secret plans"},
{"text": "to the Empire's ultimate weapon the Death Star"},
]
async_tbl = await async_db.create_table("documents_async", schema=Documents)
# ingest docs with auto-vectorization
await async_tbl.add(data)
# Create a fts index before the hybrid search
await async_tbl.create_index("text", config=FTS())
text_query = "flower moon"
# hybrid search with default re-ranker
await (await async_tbl.search("flower moon", query_type="hybrid")).to_pandas()
# --8<-- [end:basic_hybrid_search_async]
# --8<-- [start:hybrid_search_pass_vector_text_async]
vector_query = [0.1, 0.2, 0.3, 0.4, 0.5]
text_query = "flower moon"
await (
async_tbl.query()
.nearest_to(vector_query)
.nearest_to_text(text_query)
.limit(5)
.to_pandas()
)
# --8<-- [end:hybrid_search_pass_vector_text_async]
docs/web-tests/py/test_storage.py
+162
View File
@@ -0,0 +1,162 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import lancedb
import pytest
class DummyTable:
def __init__(self, name: str, storage_options: dict | None = None):
self.name = name
self.storage_options = storage_options
class DummyConnection:
def __init__(self, uri: str, options: dict):
self.uri = uri
self.options = options
self.created_tables: list[DummyTable] = []
def create_table(
self,
name: str,
data,
storage_options: dict | None = None,
):
table = DummyTable(name, storage_options=storage_options)
self.created_tables.append(table)
return table
@pytest.fixture
def fake_connect(monkeypatch):
calls: list[DummyConnection] = []
def _fake_connect(uri: str, **kwargs):
conn = DummyConnection(uri, kwargs)
calls.append(conn)
return conn
monkeypatch.setattr(lancedb, "connect", _fake_connect)
return calls
def test_storage_snippets(fake_connect):
# --8<-- [start:storage_connect_s3]
db = lancedb.connect("s3://bucket/path")
# --8<-- [end:storage_connect_s3]
# --8<-- [start:storage_connect_gcs]
db = lancedb.connect("gs://bucket/path")
# --8<-- [end:storage_connect_gcs]
# --8<-- [start:storage_connect_azure]
db = lancedb.connect("az://bucket/path")
# --8<-- [end:storage_connect_azure]
# --8<-- [start:storage_connect_timeout]
db = lancedb.connect(
"s3://bucket/path",
storage_options={"timeout": "60s"},
)
# --8<-- [end:storage_connect_timeout]
# --8<-- [start:storage_table_timeout]
table = db.create_table(
"table",
[{"a": 1, "b": 2}],
storage_options={"timeout": "60s"},
)
# --8<-- [end:storage_table_timeout]
# --8<-- [start:storage_s3_ddb]
db = lancedb.connect(
"s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
)
# --8<-- [end:storage_s3_ddb]
# --8<-- [start:storage_s3_ddb_local]
db = lancedb.connect(
"s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
storage_options={
"endpoint": "http://localhost:4566",
"dynamodb_endpoint": "http://localhost:4566",
"allow_http": "true",
},
)
# --8<-- [end:storage_s3_ddb_local]
# --8<-- [start:storage_s3_sse_kms]
db = lancedb.connect(
"s3://bucket/path",
storage_options={
"aws_server_side_encryption": "aws:kms",
"aws_sse_kms_key_id": "<your-kms-key-id-or-arn>",
},
)
# --8<-- [end:storage_s3_sse_kms]
# --8<-- [start:storage_azure_sas]
db = lancedb.connect(
"az://my-container/my-database",
storage_options={
"azure_storage_account_name": "some-account",
"azure_storage_sas_token": "<sas-token>",
},
)
# --8<-- [end:storage_azure_sas]
# --8<-- [start:storage_s3_minio]
db = lancedb.connect(
"s3://bucket/path",
storage_options={
"region": "us-east-1",
"endpoint": "http://minio:9000",
},
)
# --8<-- [end:storage_s3_minio]
# --8<-- [start:storage_s3_express]
db = lancedb.connect(
"s3://my-bucket--use1-az4--x-s3/path",
storage_options={
"region": "us-east-1",
"s3_express": "true",
},
)
# --8<-- [end:storage_s3_express]
# --8<-- [start:storage_gcs_service_account]
db = lancedb.connect(
"gs://my-bucket/my-database",
storage_options={
"service_account": "path/to/service-account.json",
},
)
# --8<-- [end:storage_gcs_service_account]
# --8<-- [start:storage_azure_account]
db = lancedb.connect(
"az://my-container/my-database",
storage_options={
"account_name": "some-account",
"account_key": "some-key",
},
)
# --8<-- [end:storage_azure_account]
# --8<-- [start:storage_tigris_connect]
db = lancedb.connect(
"s3://your-bucket/path",
storage_options={
"endpoint": "https://t3.storage.dev",
"region": "auto",
},
)
# --8<-- [end:storage_tigris_connect]
assert len(fake_connect) == 13
assert all(
conn.uri.startswith(("s3://", "gs://", "az://", "s3+ddb://"))
for conn in fake_connect
)
docs/web-tests/py/test_tables.py
+1546
View File
File diff suppressed because it is too large Load Diff
docs/web-tests/rs/Cargo.lock
Generated
+7433
View File
File diff suppressed because it is too large Load Diff
docs/web-tests/rs/Cargo.toml
+43
View File
@@ -0,0 +1,43 @@
[package]
name = "rs"
version = "0.1.0"
edition = "2024"
[dependencies]
arrow-array = "58.3.0"
arrow-json = "58.3.0"
arrow-schema = "58.3.0"
futures = "0.3.31"
futures-util = "0.3.31"
lance-namespace = "7.0.0"
lancedb = { version = "0.31.0", features = ["polars"] }
polars = ">=0.37,<0.40.0"
polars-arrow = ">=0.37,<0.40.0"
serde = { version = "1", features = ["derive"] }
serde_json = "1.0.145"
tempfile = "3.23.0"
tokio = { version = "1", features = ["macros", "rt-multi-thread"] }
[[example]]
name = "basic_usage"
path = "basic_usage.rs"
[[example]]
name = "connection"
path = "connection.rs"
[[example]]
name = "quickstart"
path = "quickstart.rs"
[[example]]
name = "tables"
path = "tables.rs"
[[example]]
name = "multimodal"
path = "multimodal.rs"
[[example]]
name = "search"
path = "search.rs"
docs/web-tests/rs/basic_usage.rs
+346
View File
@@ -0,0 +1,346 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::fs;
use std::path::PathBuf;
use std::sync::Arc;
use serde::Deserialize;
type BatchIter = Box<dyn arrow_array::RecordBatchReader + Send>;
// --8<-- [start:basic_imports]
use arrow_array::types::Float32Type;
use arrow_array::{
FixedSizeListArray, Int8Array, Int16Array, RecordBatch, RecordBatchIterator, StringArray,
StructArray,
};
use arrow_schema::{DataType, Field, FieldRef, Schema};
use futures_util::TryStreamExt;
use lancedb::database::CreateTableMode;
use lancedb::query::{ExecutableQuery, QueryBase, Select};
use lancedb::{connect, table::NewColumnTransform};
// --8<-- [end:basic_imports]
#[derive(Debug, Clone, Deserialize)]
struct Stats {
strength: i8,
courage: i8,
magic: i8,
wisdom: i8,
}
#[derive(Debug, Clone, Deserialize)]
struct Character {
id: i16,
name: String,
role: String,
description: String,
vector: [f32; 4],
stats: Stats,
}
fn camelot_schema() -> Arc<Schema> {
Arc::new(Schema::new(vec![
Field::new("id", DataType::Int16, false),
Field::new("name", DataType::Utf8, false),
Field::new("role", DataType::Utf8, false),
Field::new("description", DataType::Utf8, false),
Field::new(
"vector",
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4),
false,
),
Field::new(
"stats",
DataType::Struct(arrow_schema::Fields::from(vec![
Arc::new(Field::new("strength", DataType::Int8, false)),
Arc::new(Field::new("courage", DataType::Int8, false)),
Arc::new(Field::new("magic", DataType::Int8, false)),
Arc::new(Field::new("wisdom", DataType::Int8, false)),
])),
false,
),
]))
}
fn characters_to_record_batch(schema: Arc<Schema>, characters: &[Character]) -> RecordBatch {
let ids = Int16Array::from_iter_values(characters.iter().map(|c| c.id));
let names = StringArray::from_iter_values(characters.iter().map(|c| c.name.as_str()));
let roles = StringArray::from_iter_values(characters.iter().map(|c| c.role.as_str()));
let descriptions =
StringArray::from_iter_values(characters.iter().map(|c| c.description.as_str()));
let vectors = FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
characters
.iter()
.map(|c| Some(c.vector.iter().copied().map(Some).collect::<Vec<_>>())),
4,
);
let strength = Int8Array::from_iter_values(characters.iter().map(|c| c.stats.strength));
let courage = Int8Array::from_iter_values(characters.iter().map(|c| c.stats.courage));
let magic = Int8Array::from_iter_values(characters.iter().map(|c| c.stats.magic));
let wisdom = Int8Array::from_iter_values(characters.iter().map(|c| c.stats.wisdom));
let stats_fields: Vec<FieldRef> = vec![
Arc::new(Field::new("strength", DataType::Int8, false)),
Arc::new(Field::new("courage", DataType::Int8, false)),
Arc::new(Field::new("magic", DataType::Int8, false)),
Arc::new(Field::new("wisdom", DataType::Int8, false)),
];
let stats = StructArray::new(
stats_fields.into(),
vec![
Arc::new(strength),
Arc::new(courage),
Arc::new(magic),
Arc::new(wisdom),
],
None,
);
RecordBatch::try_new(
schema,
vec![
Arc::new(ids),
Arc::new(names),
Arc::new(roles),
Arc::new(descriptions),
Arc::new(vectors),
Arc::new(stats),
],
)
.unwrap()
}
fn characters_to_reader(schema: Arc<Schema>, characters: &[Character]) -> BatchIter {
let batch = characters_to_record_batch(schema.clone(), characters);
Box::new(RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema))
}
fn camelot_json_path() -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("..")
.join("camelot.json")
}
#[tokio::main]
async fn main() {
let temp_dir = tempfile::tempdir().unwrap();
let uri = temp_dir.path().to_str().unwrap();
let db = connect(uri).execute().await.unwrap();
// --8<-- [start:data_load]
let data: Vec<Character> =
serde_json::from_str(&fs::read_to_string(camelot_json_path()).unwrap()).unwrap();
// --8<-- [end:data_load]
let schema = camelot_schema();
// --8<-- [start:basic_create_table]
let mut table = db
.create_table("camelot", characters_to_reader(schema.clone(), &data))
.mode(CreateTableMode::Overwrite)
.execute()
.await
.unwrap();
// --8<-- [end:basic_create_table]
assert_eq!(table.count_rows(None).await.unwrap(), 8);
// --8<-- [start:basic_open_table]
table = db.open_table("camelot").execute().await.unwrap();
// --8<-- [end:basic_open_table]
// --8<-- [start:basic_create_empty_table]
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int16, false),
Field::new("name", DataType::Utf8, false),
Field::new("role", DataType::Utf8, false),
Field::new("description", DataType::Utf8, false),
Field::new(
"vector",
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4),
false,
),
Field::new(
"stats",
DataType::Struct(arrow_schema::Fields::from(vec![
Arc::new(Field::new("strength", DataType::Int8, false)),
Arc::new(Field::new("courage", DataType::Int8, false)),
Arc::new(Field::new("magic", DataType::Int8, false)),
Arc::new(Field::new("wisdom", DataType::Int8, false)),
])),
false,
),
]));
db.create_empty_table("camelot_empty", schema)
.mode(CreateTableMode::Overwrite)
.execute()
.await
.unwrap();
// --8<-- [end:basic_create_empty_table]
db.drop_table("camelot_empty", &[]).await.unwrap();
// --8<-- [start:basic_add_data]
let magical_characters = vec![
Character {
id: 9,
name: "Morgan le Fay".to_string(),
role: "Sorceress".to_string(),
description: "A powerful enchantress, Arthur's half-sister, and a complex figure who oscillates between aiding and opposing Camelot.".to_string(),
vector: [0.10, 0.84, 0.25, 0.70],
stats: Stats {
strength: 2,
courage: 3,
magic: 5,
wisdom: 4,
},
},
Character {
id: 10,
name: "The Lady of the Lake".to_string(),
role: "Mystical Guardian".to_string(),
description: "A mysterious supernatural figure associated with Avalon, known for giving Arthur the sword Excalibur.".to_string(),
vector: [0.00, 0.90, 0.58, 0.88],
stats: Stats {
strength: 2,
courage: 3,
magic: 5,
wisdom: 5,
},
},
];
table
.add(characters_to_reader(camelot_schema(), &magical_characters))
.execute()
.await
.unwrap();
// --8<-- [end:basic_add_data]
// --8<-- [start:basic_vector_search]
let query_vector = [0.03, 0.85, 0.61, 0.90];
let result = table
.query()
.nearest_to(&query_vector)
.unwrap()
.limit(5)
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
println!("{result:?}");
// --8<-- [end:basic_vector_search]
// --8<-- [start:basic_add_columns]
table
.add_columns(
NewColumnTransform::SqlExpressions(vec![(
"power".to_string(),
"cast(((stats.strength + stats.courage + stats.magic + stats.wisdom) / 4.0) as float)"
.to_string(),
)]),
None,
)
.await
.unwrap();
// --8<-- [end:basic_add_columns]
// --8<-- [start:basic_vector_search_q1]
// Who are the characters similar to "wizard"?
let query_vector_1 = [0.03, 0.85, 0.61, 0.90];
let r1 = table
.query()
.nearest_to(&query_vector_1)
.unwrap()
.limit(5)
.select(Select::Columns(vec![
"name".to_string(),
"role".to_string(),
"description".to_string(),
]))
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
println!("{r1:?}");
// --8<-- [end:basic_vector_search_q1]
// --8<-- [start:basic_vector_search_q2]
// Who are the characters similar to "wizard" with high magic stats?
let query_vector_2 = [0.03, 0.85, 0.61, 0.90];
let r2 = table
.query()
.nearest_to(&query_vector_2)
.unwrap()
.only_if("stats.magic > 3")
.select(Select::Columns(vec![
"name".to_string(),
"role".to_string(),
"description".to_string(),
]))
.limit(5)
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
println!("{r2:?}");
// --8<-- [end:basic_vector_search_q2]
// --8<-- [start:basic_vector_search_q3]
// Who are the strongest characters?
let r3 = table
.query()
.only_if("stats.strength > 3")
.select(Select::Columns(vec![
"name".to_string(),
"role".to_string(),
"description".to_string(),
]))
.limit(5)
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
println!("{r3:?}");
// --8<-- [end:basic_vector_search_q3]
// --8<-- [start:basic_vector_search_q4]
// Who are the strongest characters?
let r4 = table
.query()
.select(Select::Columns(vec![
"name".to_string(),
"role".to_string(),
"description".to_string(),
"power".to_string(),
]))
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
println!("{r4:?}");
// --8<-- [end:basic_vector_search_q4]
// --8<-- [start:basic_drop_columns]
table.drop_columns(&["power"]).await.unwrap();
// --8<-- [end:basic_drop_columns]
// --8<-- [start:basic_delete_rows]
table.delete("role = 'Traitor Knight'").await.unwrap();
// --8<-- [end:basic_delete_rows]
// --8<-- [start:basic_drop_table]
db.drop_table("camelot", &[]).await.unwrap();
// --8<-- [end:basic_drop_table]
}
docs/web-tests/rs/bedrock.rs
+92
View File
@@ -0,0 +1,92 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::{iter::once, sync::Arc};
use arrow_array::{Float64Array, Int32Array, RecordBatch, RecordBatchIterator, StringArray};
use arrow_schema::{DataType, Field, Schema};
use aws_config::Region;
use aws_sdk_bedrockruntime::Client;
use futures::StreamExt;
use lancedb::{
arrow::IntoArrow,
connect,
embeddings::{bedrock::BedrockEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},
query::{ExecutableQuery, QueryBase},
Result,
};
#[tokio::main]
async fn main() -> Result<()> {
let tempdir = tempfile::tempdir().unwrap();
let tempdir = tempdir.path().to_str().unwrap();
// create Bedrock embedding function
let region: String = "us-east-1".to_string();
let config = aws_config::defaults(aws_config::BehaviorVersion::latest())
.region(Region::new(region))
.load()
.await;
let embedding = Arc::new(BedrockEmbeddingFunction::new(
Client::new(&config), // AWS Region
));
let db = connect(tempdir).execute().await?;
db.embedding_registry()
.register("bedrock", embedding.clone())?;
let table = db
.create_table("vectors", make_data())
.add_embedding(EmbeddingDefinition::new(
"text",
"bedrock",
Some("embeddings"),
))?
.execute()
.await?;
// execute vector search
let query = Arc::new(StringArray::from_iter_values(once("something warm")));
let query_vector = embedding.compute_query_embeddings(query)?;
let mut results = table
.vector_search(query_vector)?
.limit(1)
.execute()
.await?;
let rb = results.next().await.unwrap()?;
let out = rb
.column_by_name("text")
.unwrap()
.as_any()
.downcast_ref::<StringArray>()
.unwrap();
let text = out.iter().next().unwrap().unwrap();
println!("Closest match: {}", text);
Ok(())
}
fn make_data() -> impl IntoArrow {
let schema = Schema::new(vec![
Field::new("id", DataType::Int32, true),
Field::new("text", DataType::Utf8, false),
Field::new("price", DataType::Float64, false),
]);
let id = Int32Array::from(vec![1, 2, 3, 4]);
let text = StringArray::from_iter_values(vec![
"Black T-Shirt",
"Leather Jacket",
"Winter Parka",
"Hooded Sweatshirt",
]);
let price = Float64Array::from(vec![10.0, 50.0, 100.0, 30.0]);
let schema = Arc::new(schema);
let rb = RecordBatch::try_new(
schema.clone(),
vec![Arc::new(id), Arc::new(text), Arc::new(price)],
)
.unwrap();
Box::new(RecordBatchIterator::new(vec![Ok(rb)], schema))
}
docs/web-tests/rs/connection.rs
+134
View File
@@ -0,0 +1,134 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::path::PathBuf;
use lancedb::connect;
// --8<-- [start:connect]
async fn connect_example(uri: &str) {
let db = connect(uri).execute().await.unwrap();
let _ = db;
}
// --8<-- [end:connect]
#[tokio::main]
async fn main() {
let temp_dir = tempfile::tempdir().unwrap();
let uri = temp_dir.path().join("ex_lancedb");
connect_example(uri.to_str().unwrap()).await;
// Keep the enterprise snippet in this file, but don't run it in CI.
let _ = connect_enterprise_quickstart_config();
let _ = connect_object_storage_config();
}
fn connect_enterprise_quickstart_config() -> (String, String, String, String) {
// --8<-- [start:connect_enterprise_quickstart]
let uri = "db://your-database-uri";
let api_key = "your-api-key";
let region = "us-east-1";
let host_override = "https://your-enterprise-endpoint.com";
// --8<-- [end:connect_enterprise_quickstart]
(
uri.to_string(),
api_key.to_string(),
region.to_string(),
host_override.to_string(),
)
}
fn connect_object_storage_config() -> &'static str {
// --8<-- [start:connect_object_storage]
let uri = "s3://your-bucket/path";
// You can also use "gs://your-bucket/path" or "az://your-container/path".
// --8<-- [end:connect_object_storage]
uri
}
async fn namespace_table_ops_example(uri: &str) -> lancedb::Result<()> {
// --8<-- [start:namespace_table_ops]
let conn = connect(uri).execute().await?;
let search_namespace = vec!["prod".to_string(), "search".to_string()];
let recommendations_namespace = vec!["prod".to_string(), "recommendations".to_string()];
let schema = std::sync::Arc::new(arrow_schema::Schema::new(vec![
arrow_schema::Field::new("id", arrow_schema::DataType::Int64, false),
]));
conn.create_empty_table("user", schema.clone())
.namespace(search_namespace.clone())
.execute()
.await?;
conn.create_empty_table("user", schema)
.namespace(recommendations_namespace.clone())
.execute()
.await?;
let search_table_names = conn
.table_names()
.namespace(search_namespace)
.execute()
.await?;
let recommendation_table_names = conn
.table_names()
.namespace(recommendations_namespace)
.execute()
.await?;
println!("{search_table_names:?}"); // ["user"]
println!("{recommendation_table_names:?}"); // ["user"]
// --8<-- [end:namespace_table_ops]
Ok(())
}
async fn namespace_admin_ops_example() -> lancedb::Result<()> {
// --8<-- [start:namespace_admin_ops]
let mut properties = std::collections::HashMap::new();
properties.insert("root".to_string(), "./local_lancedb".to_string());
let db = lancedb::connect_namespace("dir", properties).execute().await?;
let namespace = vec!["prod".to_string(), "search".to_string()];
db.create_namespace(lance_namespace::models::CreateNamespaceRequest {
id: Some(vec!["prod".to_string()]),
..Default::default()
})
.await?;
db.create_namespace(lance_namespace::models::CreateNamespaceRequest {
id: Some(namespace.clone()),
..Default::default()
})
.await?;
let child_namespaces = db
.list_namespaces(lance_namespace::models::ListNamespacesRequest {
id: Some(vec!["prod".to_string()]),
..Default::default()
})
.await?;
println!(
"Child namespaces under {:?}: {:?}",
namespace, child_namespaces
);
// Child namespaces under ["prod", "search"]: ["search"]
db.drop_namespace(lance_namespace::models::DropNamespaceRequest {
id: Some(namespace.clone()),
..Default::default()
})
.await?;
db.drop_namespace(lance_namespace::models::DropNamespaceRequest {
id: Some(vec!["prod".to_string()]),
..Default::default()
})
.await?;
// --8<-- [end:namespace_admin_ops]
Ok(())
}
#[allow(dead_code)]
fn repo_root() -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..")
}
docs/web-tests/rs/embedding.rs
+182
View File
@@ -0,0 +1,182 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// --8<-- [start:openai_embeddings]
use std::{iter::once, sync::Arc};
use arrow_array::{record_batch, StringArray};
use arrow_schema::{DataType, Field, Schema};
use futures::StreamExt;
use lancedb::{
connect,
embeddings::{openai::OpenAIEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},
query::{ExecutableQuery, QueryBase},
Result,
};
#[tokio::main]
async fn main() -> Result<()> {
let db = connect("./mydb").execute().await?;
let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY is not set");
let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model(
api_key,
"text-embedding-3-large",
)?);
db.embedding_registry().register("openai", embedding.clone())?;
let schema = Arc::new(Schema::new(vec![Field::new("text", DataType::Utf8, false)]));
let table = db
.create_empty_table("mytable", schema)
.add_embedding(EmbeddingDefinition::new("text", "openai", Some("vector")))?
.execute()
.await?;
table
.add(record_batch!(("text", Utf8, ["This is a test.", "Another example."]))?)
.execute()
.await?;
let query = Arc::new(StringArray::from_iter_values(once("test example")));
let query_vector = embedding.compute_query_embeddings(query)?;
let mut results = table.vector_search(query_vector)?.limit(5).execute().await?;
while let Some(batch) = results.next().await {
println!("{:?}", batch?);
}
Ok(())
}
// --8<-- [end:openai_embeddings]
// --8<-- [start:create_embedding_function]
use std::sync::Arc;
use lancedb::embeddings::openai::OpenAIEmbeddingFunction;
let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY is not set");
let embedding = Arc::new(
OpenAIEmbeddingFunction::new_with_model(api_key, "text-embedding-3-small")
.expect("failed to create OpenAI embedding function"),
);
// --8<-- [end:create_embedding_function]
// --8<-- [start:manual_query_embeddings]
use std::{iter::once, sync::Arc};
use arrow_array::{record_batch, StringArray};
use arrow_schema::{DataType, Field, Schema};
use futures::StreamExt;
use lancedb::{
connect,
embeddings::{openai::OpenAIEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},
query::{ExecutableQuery, QueryBase},
Result,
};
#[tokio::main]
async fn main() -> Result<()> {
let db = connect("./mydb").execute().await?;
let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY is not set");
let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model(
api_key,
"text-embedding-3-large",
)?);
db.embedding_registry().register("openai", embedding.clone())?;
let schema = Arc::new(Schema::new(vec![Field::new("text", DataType::Utf8, false)]));
let table = db
.create_empty_table("mytable", schema)
.add_embedding(EmbeddingDefinition::new("text", "openai", Some("vector")))?
.execute()
.await?;
table
.add(record_batch!(("text", Utf8, ["This is a test.", "Another example."]))?)
.execute()
.await?;
// Manually generate embeddings for the query (Enterprise path)
let query = Arc::new(StringArray::from_iter_values(once("test example")));
let query_vector = embedding.compute_query_embeddings(query)?;
// --8<-- [start:manual_query_search]
// query_vector is assumed to already be generated by your embedding function
let mut results = table.vector_search(query_vector)?.limit(5).execute().await?;
while let Some(batch) = results.next().await {
println!("{:?}", batch?);
}
// --8<-- [end:manual_query_search]
Ok(())
}
// --8<-- [end:manual_query_embeddings]
// --8<-- [start:embedding_function]
use std::{borrow::Cow, sync::Arc};
use arrow_array::{Array, FixedSizeListArray, Float32Array};
use arrow_schema::{DataType, Field, Schema};
use lancedb::{
connect,
embeddings::{EmbeddingDefinition, EmbeddingFunction},
Result,
};
#[derive(Debug, Clone)]
struct MyTextEmbedder {
dim: usize,
}
impl EmbeddingFunction for MyTextEmbedder {
fn name(&self) -> &str {
"my-embedder"
}
fn source_type(&self) -> Result<Cow<'_, DataType>> {
Ok(Cow::Owned(DataType::Utf8))
}
fn dest_type(&self) -> Result<Cow<'_, DataType>> {
Ok(Cow::Owned(DataType::new_fixed_size_list(
DataType::Float32,
self.dim as i32,
true,
)))
}
fn compute_source_embeddings(&self, source: Arc<dyn Array>) -> Result<Arc<dyn Array>> {
let values = Arc::new(Float32Array::from(vec![1.0f32; source.len() * self.dim]));
let field = Arc::new(Field::new("item", DataType::Float32, true));
Ok(Arc::new(FixedSizeListArray::new(
field,
self.dim as i32,
values,
None,
)))
}
fn compute_query_embeddings(&self, _input: Arc<dyn Array>) -> Result<Arc<dyn Array>> {
unimplemented!()
}
}
#[tokio::main]
async fn main() -> Result<()> {
let db = connect("./mydb").execute().await?;
db.embedding_registry()
.register("my-embedder", Arc::new(MyTextEmbedder { dim: 3 }))?;
let schema = Arc::new(Schema::new(vec![Field::new("text", DataType::Utf8, false)]));
db.create_empty_table("mytable", schema)
.add_embedding(EmbeddingDefinition::new(
"text",
"my-embedder",
Some("vector"),
))?
.execute()
.await?;
Ok(())
}
// --8<-- [end:embedding_function]
docs/web-tests/rs/full_text_search.rs
+103
View File
@@ -0,0 +1,103 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::sync::Arc;
use arrow_array::{Int32Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray};
use arrow_schema::{DataType, Field, Schema};
use futures::TryStreamExt;
use lance_index::scalar::FullTextSearchQuery;
use lancedb::connection::Connection;
use lancedb::index::scalar::FtsIndexBuilder;
use lancedb::index::Index;
use lancedb::query::{ExecutableQuery, QueryBase};
use lancedb::{connect, Result, Table};
use rand::random;
#[tokio::main]
async fn main() -> Result<()> {
if std::path::Path::new("data").exists() {
std::fs::remove_dir_all("data").unwrap();
}
let uri = "data/sample-lancedb";
let db = connect(uri).execute().await?;
let tbl = create_table(&db).await?;
create_index(&tbl).await?;
search_index(&tbl).await?;
Ok(())
}
fn create_some_records() -> Result<Box<dyn RecordBatchReader + Send>> {
const TOTAL: usize = 1000;
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int32, false),
Field::new("doc", DataType::Utf8, true),
]));
let words = random_word::all(random_word::Lang::En)
.iter()
.step_by(1024)
.take(500)
.copied()
.collect::<Vec<_>>();
let n_terms = 3;
let batches = RecordBatchIterator::new(
vec![RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Int32Array::from_iter_values(0..TOTAL as i32)),
Arc::new(StringArray::from_iter_values((0..TOTAL).map(|_| {
(0..n_terms)
.map(|_| words[random::<usize>() % words.len()])
.collect::<Vec<_>>()
.join(" ")
}))),
],
)
.unwrap()]
.into_iter()
.map(Ok),
schema.clone(),
);
Ok(Box::new(batches))
}
async fn create_table(db: &Connection) -> Result<Table> {
let initial_data: Box<dyn RecordBatchReader + Send> = create_some_records()?;
let tbl = db.create_table("my_table", initial_data).execute().await?;
Ok(tbl)
}
async fn create_index(table: &Table) -> Result<()> {
table
.create_index(&["doc"], Index::FTS(FtsIndexBuilder::default()))
.execute()
.await?;
Ok(())
}
async fn search_index(table: &Table) -> Result<()> {
let words = random_word::all(random_word::Lang::En)
.iter()
.step_by(1024)
.take(500)
.copied()
.collect::<Vec<_>>();
let query = words[0].to_owned();
println!("Searching for: {}", query);
let mut results = table
.query()
.full_text_search(FullTextSearchQuery::new(words[0].to_owned()))
.select(lancedb::query::Select::Columns(vec!["doc".to_owned()]))
.limit(10)
.execute()
.await?;
while let Some(batch) = results.try_next().await? {
println!("{:?}", batch);
}
Ok(())
}
docs/web-tests/rs/ivf_pq.rs
+154
View File
@@ -0,0 +1,154 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
//! This example demonstrates setting advanced parameters when building an IVF PQ index
//!
//! Snippets from this example are used in the documentation on ANN indices.
use std::sync::Arc;
use arrow_array::types::Float32Type;
use arrow_array::{
FixedSizeListArray, Int32Array, RecordBatch, RecordBatchIterator, RecordBatchReader,
};
use arrow_schema::{DataType, Field, Schema};
use futures::TryStreamExt;
use lancedb::connection::Connection;
use lancedb::index::vector::IvfPqIndexBuilder;
use lancedb::index::Index;
use lancedb::query::{ExecutableQuery, QueryBase};
use lancedb::{connect, DistanceType, Result, Table};
#[tokio::main]
async fn main() -> Result<()> {
if std::path::Path::new("data").exists() {
std::fs::remove_dir_all("data").unwrap();
}
let uri = "data/sample-lancedb";
let db = connect(uri).execute().await?;
let tbl = create_table(&db).await?;
create_index(&tbl).await?;
search_index(&tbl).await?;
Ok(())
}
fn create_some_records() -> Result<Box<dyn RecordBatchReader + Send>> {
const TOTAL: usize = 1000;
const DIM: usize = 128;
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int32, false),
Field::new(
"vector",
DataType::FixedSizeList(
Arc::new(Field::new("item", DataType::Float32, true)),
DIM as i32,
),
true,
),
]));
// Create a RecordBatch stream.
let batches = RecordBatchIterator::new(
vec![RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Int32Array::from_iter_values(0..TOTAL as i32)),
Arc::new(
FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
(0..TOTAL).map(|_| Some(vec![Some(1.0); DIM])),
DIM as i32,
),
),
],
)
.unwrap()]
.into_iter()
.map(Ok),
schema.clone(),
);
Ok(Box::new(batches))
}
async fn create_table(db: &Connection) -> Result<Table> {
let initial_data: Box<dyn RecordBatchReader + Send> = create_some_records()?;
let tbl = db
.create_table("my_table", initial_data)
.execute()
.await
.unwrap();
Ok(tbl)
}
async fn create_index(table: &Table) -> Result<()> {
// --8<-- [start:create_index]
// For this example, `table` is a lancedb::Table with a column named
// "vector" that is a vector column with dimension 128.
// By default, if the column "vector" appears to be a vector column,
// then an IVF_PQ index with reasonable defaults is created.
table
.create_index(&["vector"], Index::Auto)
.execute()
.await?;
// For advanced cases, it is also possible to specifically request an
// IVF_PQ index and provide custom parameters.
table
.create_index(
&["vector"],
Index::IvfPq(
// Here we specify advanced indexing parameters. In this case
// we are creating an index that my have better recall than the
// default but is also larger and slower.
IvfPqIndexBuilder::default()
// This overrides the default distance type of l2
.distance_type(DistanceType::Cosine)
// With 1000 rows this have been ~31 by default
.num_partitions(50)
// With dimension 128 this would have been 8 by default
.num_sub_vectors(16),
),
)
.execute()
.await?;
// --8<-- [end:create_index]
Ok(())
}
async fn search_index(table: &Table) -> Result<()> {
// --8<-- [start:search1]
let query_vector = [1.0; 128];
// By default the index will find the 10 closest results using default
// search parameters that give a reasonable tradeoff between accuracy
// and search latency
let mut results = table
.vector_search(&query_vector)?
// Note: you should always set the distance_type to match the value used
// to train the index
.distance_type(DistanceType::Cosine)
.execute()
.await?;
while let Some(batch) = results.try_next().await? {
println!("{:?}", batch);
}
// We can also provide custom search parameters. Here we perform a
// slower but more accurate search
let mut results = table
.vector_search(&query_vector)?
.distance_type(DistanceType::Cosine)
// Override the default of 10 to get more rows
.limit(15)
// Override the default of 20 to search more partitions
.nprobes(30)
// Override the default of None to apply a refine step
.refine_factor(1)
.execute()
.await?;
while let Some(batch) = results.try_next().await? {
println!("{:?}", batch);
}
Ok(())
// --8<-- [end:search1]
}
docs/web-tests/rs/multimodal.rs
+178
View File
@@ -0,0 +1,178 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// --8<-- [start:multimodal_imports]
use std::collections::HashMap;
use std::sync::Arc;
use arrow_array::types::Float32Type;
use arrow_array::{
BinaryArray, FixedSizeListArray, Int32Array, Int64Array, LargeBinaryArray, RecordBatch,
RecordBatchIterator, StringArray,
};
use arrow_schema::{DataType, Field, Schema};
use futures_util::TryStreamExt;
use lancedb::connect;
use lancedb::database::CreateTableMode;
use lancedb::query::{ExecutableQuery, QueryBase};
// --8<-- [end:multimodal_imports]
#[tokio::main]
async fn main() {
let temp_dir = tempfile::tempdir().unwrap();
let db_uri = temp_dir.path().to_str().unwrap().to_string();
let db = connect(&db_uri).execute().await.unwrap();
// --8<-- [start:create_dummy_data]
let create_dummy_image = |color: u8| -> Vec<u8> {
let mut png_like = vec![137, 80, 78, 71, 13, 10, 26, 10];
png_like.push(color);
png_like
};
let data = vec![
(
1_i32,
"red_square.png",
vec![0.1_f32; 128],
create_dummy_image(1),
"red",
),
(
2_i32,
"blue_square.png",
vec![0.2_f32; 128],
create_dummy_image(2),
"blue",
),
];
// --8<-- [end:create_dummy_data]
// --8<-- [start:define_schema]
let schema = Schema::new(vec![
Field::new("id", DataType::Int32, false),
Field::new("filename", DataType::Utf8, false),
Field::new(
"vector",
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 128),
false,
),
Field::new("image_blob", DataType::Binary, false),
Field::new("label", DataType::Utf8, false),
]);
// --8<-- [end:define_schema]
// --8<-- [start:ingest_data]
let schema = Arc::new(schema);
let image_batch = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Int32Array::from_iter_values(data.iter().map(|row| row.0))),
Arc::new(StringArray::from_iter_values(data.iter().map(|row| row.1))),
Arc::new(
FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
data.iter()
.map(|row| Some(row.2.iter().copied().map(Some).collect::<Vec<_>>())),
128,
),
),
Arc::new(BinaryArray::from_iter_values(
data.iter().map(|row| row.3.as_slice()),
)),
Arc::new(StringArray::from_iter_values(data.iter().map(|row| row.4))),
],
)
.unwrap();
let image_reader: Box<dyn arrow_array::RecordBatchReader + Send> =
Box::new(RecordBatchIterator::new(vec![Ok(image_batch)].into_iter(), schema.clone()));
let table = db
.create_table("images", image_reader)
.mode(CreateTableMode::Overwrite)
.execute()
.await
.unwrap();
// --8<-- [end:ingest_data]
assert_eq!(table.count_rows(None).await.unwrap(), 2);
// --8<-- [start:search_data]
let query_vector = vec![0.1_f32; 128];
let results = table
.query()
.nearest_to(query_vector)
.unwrap()
.limit(1)
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
// --8<-- [end:search_data]
// --8<-- [start:process_results]
for batch in &results {
let filenames = batch
.column_by_name("filename")
.unwrap()
.as_any()
.downcast_ref::<StringArray>()
.unwrap();
let images = batch
.column_by_name("image_blob")
.unwrap()
.as_any()
.downcast_ref::<BinaryArray>()
.unwrap();
for row in 0..batch.num_rows() {
let image_bytes = images.value(row);
println!(
"Retrieved image: {}, Byte length: {}",
filenames.value(row),
image_bytes.len()
);
}
}
// --8<-- [end:process_results]
let search_rows: usize = results.iter().map(|batch| batch.num_rows()).sum();
assert_eq!(search_rows, 1);
// --8<-- [start:blob_api_schema]
let blob_metadata = HashMap::from([(
"lance-encoding:blob".to_string(),
"true".to_string(),
)]);
let blob_schema = Schema::new(vec![
Field::new("id", DataType::Int64, false),
Field::new("video", DataType::LargeBinary, true).with_metadata(blob_metadata),
]);
// --8<-- [end:blob_api_schema]
// --8<-- [start:blob_api_ingest]
let blob_rows = vec![
(1_i64, b"fake_video_bytes_1".to_vec()),
(2_i64, b"fake_video_bytes_2".to_vec()),
];
let blob_schema = Arc::new(blob_schema);
let blob_batch = RecordBatch::try_new(
blob_schema.clone(),
vec![
Arc::new(Int64Array::from_iter_values(blob_rows.iter().map(|row| row.0))),
Arc::new(LargeBinaryArray::from_iter_values(
blob_rows.iter().map(|row| row.1.as_slice()),
)),
],
)
.unwrap();
let blob_reader: Box<dyn arrow_array::RecordBatchReader + Send> =
Box::new(RecordBatchIterator::new(vec![Ok(blob_batch)].into_iter(), blob_schema));
let blob_table = db
.create_table("videos", blob_reader)
.mode(CreateTableMode::Overwrite)
.execute()
.await
.unwrap();
// --8<-- [end:blob_api_ingest]
assert_eq!(blob_table.count_rows(None).await.unwrap(), 2);
}
docs/web-tests/rs/openai.rs
+89
View File
@@ -0,0 +1,89 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// --8<-- [start:imports]
use std::{iter::once, sync::Arc};
use arrow_array::{Float64Array, Int32Array, RecordBatch, RecordBatchIterator, StringArray};
use arrow_schema::{DataType, Field, Schema};
use futures::StreamExt;
use lancedb::{
arrow::IntoArrow,
connect,
embeddings::{openai::OpenAIEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},
query::{ExecutableQuery, QueryBase},
Result,
};
// --8<-- [end:imports]
// --8<-- [start:openai_embeddings]
#[tokio::main]
async fn main() -> Result<()> {
let tempdir = tempfile::tempdir().unwrap();
let tempdir = tempdir.path().to_str().unwrap();
let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY is not set");
let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model(
api_key,
"text-embedding-3-large",
)?);
let db = connect(tempdir).execute().await?;
db.embedding_registry()
.register("openai", embedding.clone())?;
let table = db
.create_table("vectors", make_data())
.add_embedding(EmbeddingDefinition::new(
"text",
"openai",
Some("embeddings"),
))?
.execute()
.await?;
let query = Arc::new(StringArray::from_iter_values(once("something warm")));
let query_vector = embedding.compute_query_embeddings(query)?;
let mut results = table
.vector_search(query_vector)?
.limit(1)
.execute()
.await?;
let rb = results.next().await.unwrap()?;
let out = rb
.column_by_name("text")
.unwrap()
.as_any()
.downcast_ref::<StringArray>()
.unwrap();
let text = out.iter().next().unwrap().unwrap();
println!("Closest match: {}", text);
Ok(())
}
// --8<-- [end:openai_embeddings]
fn make_data() -> impl IntoArrow {
let schema = Schema::new(vec![
Field::new("id", DataType::Int32, true),
Field::new("text", DataType::Utf8, false),
Field::new("price", DataType::Float64, false),
]);
let id = Int32Array::from(vec![1, 2, 3, 4]);
let text = StringArray::from_iter_values(vec![
"Black T-Shirt",
"Leather Jacket",
"Winter Parka",
"Hooded Sweatshirt",
]);
let price = Float64Array::from(vec![10.0, 50.0, 100.0, 30.0]);
let schema = Arc::new(schema);
let rb = RecordBatch::try_new(
schema.clone(),
vec![Arc::new(id), Arc::new(text), Arc::new(price)],
)
.unwrap();
Box::new(RecordBatchIterator::new(vec![Ok(rb)], schema))
}
docs/web-tests/rs/quickstart.rs
+347
View File
@@ -0,0 +1,347 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::sync::Arc;
use arrow_array::types::Float32Type;
use arrow_array::{
FixedSizeListArray, Int8Array, LargeStringArray, RecordBatch, RecordBatchIterator, StructArray,
};
use arrow_schema::{DataType, Field, FieldRef, Schema};
use lancedb::arrow::IntoPolars;
use lancedb::database::CreateTableMode;
use lancedb::query::{ExecutableQuery, QueryBase, Select};
use lancedb::{connect, table::NewColumnTransform};
use polars::prelude::DataFrame;
use serde::{Deserialize, Serialize};
// --8<-- [start:quickstart_define_struct]
// Define structs representing the data schema
#[derive(Debug, Clone, Serialize, Deserialize)]
struct Stats {
strength: i8,
magic: i8,
leadership: i8,
wisdom: i8,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
struct Character {
id: String,
name: String,
role: String,
description: String,
stats: Stats,
vector: [f32; 4],
}
fn characters_schema() -> Arc<Schema> {
Arc::new(Schema::new(vec![
Field::new("id", DataType::LargeUtf8, false),
Field::new("name", DataType::LargeUtf8, false),
Field::new("role", DataType::LargeUtf8, false),
Field::new("description", DataType::LargeUtf8, false),
Field::new(
"stats",
DataType::Struct(arrow_schema::Fields::from(vec![
Arc::new(Field::new("strength", DataType::Int8, false)),
Arc::new(Field::new("magic", DataType::Int8, false)),
Arc::new(Field::new("leadership", DataType::Int8, false)),
Arc::new(Field::new("wisdom", DataType::Int8, false)),
])),
false,
),
Field::new(
"vector",
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4),
false,
),
]))
}
// --8<-- [end:quickstart_define_struct]
type BatchIter = Box<dyn arrow_array::RecordBatchReader + Send>;
fn characters_to_reader(schema: Arc<Schema>, rows: &[Character]) -> BatchIter {
let ids = LargeStringArray::from_iter_values(rows.iter().map(|row| row.id.as_str()));
let names = LargeStringArray::from_iter_values(rows.iter().map(|row| row.name.as_str()));
let roles = LargeStringArray::from_iter_values(rows.iter().map(|row| row.role.as_str()));
let descriptions =
LargeStringArray::from_iter_values(rows.iter().map(|row| row.description.as_str()));
let strength = Int8Array::from_iter_values(rows.iter().map(|row| row.stats.strength));
let magic = Int8Array::from_iter_values(rows.iter().map(|row| row.stats.magic));
let leadership = Int8Array::from_iter_values(rows.iter().map(|row| row.stats.leadership));
let wisdom = Int8Array::from_iter_values(rows.iter().map(|row| row.stats.wisdom));
let stats_fields: Vec<FieldRef> = vec![
Arc::new(Field::new("strength", DataType::Int8, false)),
Arc::new(Field::new("magic", DataType::Int8, false)),
Arc::new(Field::new("leadership", DataType::Int8, false)),
Arc::new(Field::new("wisdom", DataType::Int8, false)),
];
let stats = StructArray::new(
stats_fields.into(),
vec![
Arc::new(strength),
Arc::new(magic),
Arc::new(leadership),
Arc::new(wisdom),
],
None,
);
let vectors = FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
rows.iter()
.map(|row| Some(row.vector.iter().copied().map(Some).collect::<Vec<_>>())),
4,
);
let batch = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(ids),
Arc::new(names),
Arc::new(roles),
Arc::new(descriptions),
Arc::new(stats),
Arc::new(vectors),
],
)
.unwrap();
Box::new(RecordBatchIterator::new(
vec![Ok(batch)].into_iter(),
schema,
))
}
#[tokio::main]
async fn main() {
let temp_dir = tempfile::tempdir().unwrap();
let uri = temp_dir.path().to_str().unwrap();
let db = connect(uri).execute().await.unwrap();
// --8<-- [start:quickstart_data]
let data = vec![
Character {
id: "1".to_string(),
name: "King Arthur".to_string(),
role: "King".to_string(),
description: "Leader of Camelot and wielder of Excalibur.".to_string(),
stats: Stats {
strength: 4,
magic: 1,
leadership: 5,
wisdom: 4,
},
vector: [0.7, 0.1, 0.9, 0.7],
},
Character {
id: "2".to_string(),
name: "Merlin".to_string(),
role: "Wizard".to_string(),
description: "Advisor and prophet with deep magical knowledge.".to_string(),
stats: Stats {
strength: 2,
magic: 5,
leadership: 4,
wisdom: 5,
},
vector: [0.2, 0.9, 0.4, 0.9],
},
Character {
id: "3".to_string(),
name: "Sir Lancelot".to_string(),
role: "Knight".to_string(),
description: "Legendary knight known for courage and combat skill.".to_string(),
stats: Stats {
strength: 5,
magic: 1,
leadership: 3,
wisdom: 3,
},
vector: [0.9, 0.1, 0.5, 0.4],
},
];
// --8<-- [end:quickstart_data]
// --8<-- [start:quickstart_create_table]
let schema = characters_schema();
let table = db
.create_table("characters", characters_to_reader(schema.clone(), &data))
.mode(CreateTableMode::Overwrite)
.execute()
.await
.unwrap();
// --8<-- [end:quickstart_create_table]
assert_eq!(table.count_rows(None).await.unwrap(), 3);
db.drop_table("characters", &[]).await.unwrap();
// --8<-- [start:quickstart_create_table_no_overwrite]
let table = db
.create_table("characters", characters_to_reader(schema.clone(), &data))
.execute()
.await
.unwrap();
// --8<-- [end:quickstart_create_table_no_overwrite]
assert_eq!(table.count_rows(None).await.unwrap(), 3);
// --8<-- [start:quickstart_vector_search_1]
// Search for examples similar to a "wise magical advisor"
let query_vector = [0.2, 0.8, 0.4, 0.9];
let result: DataFrame = table
.query()
.nearest_to(&query_vector)
.unwrap()
.select(Select::Columns(vec![
"name".to_string(),
"role".to_string(),
"description".to_string(),
"_distance".to_string(),
]))
.limit(2)
.execute()
.await
.unwrap()
.into_polars()
.await
.unwrap();
println!("{result:?}");
// --8<-- [end:quickstart_vector_search_1]
let name_col = result.column("name").unwrap().str().unwrap();
assert_eq!(name_col.get(0).unwrap(), "Merlin");
// --8<-- [start:quickstart_curate_with_metadata]
let curated: DataFrame = table
.query()
.nearest_to(&query_vector)
.unwrap()
.only_if("stats.magic >= 4")
.select(Select::Columns(vec![
"name".to_string(),
"role".to_string(),
"description".to_string(),
"_distance".to_string(),
]))
.limit(2)
.execute()
.await
.unwrap()
.into_polars()
.await
.unwrap();
println!("{curated:?}");
// --8<-- [end:quickstart_curate_with_metadata]
let curated_name_col = curated.column("name").unwrap().str().unwrap();
assert_eq!(curated_name_col.get(0).unwrap(), "Merlin");
// --8<-- [start:quickstart_output_array]
let result: DataFrame = table
.query()
.nearest_to(&query_vector)
.unwrap()
.select(Select::Columns(vec![
"name".to_string(),
"role".to_string(),
"description".to_string(),
"_distance".to_string(),
]))
.limit(2)
.execute()
.await
.unwrap()
.into_polars()
.await
.unwrap();
println!("{result:?}");
// --8<-- [end:quickstart_output_array]
let name_col = result.column("name").unwrap().str().unwrap();
assert_eq!(name_col.get(0).unwrap(), "Merlin");
// --8<-- [start:quickstart_add_feature]
table
.add_columns(
NewColumnTransform::SqlExpressions(vec![(
"power_score".to_string(),
"cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)"
.to_string(),
)]),
None,
)
.await
.unwrap();
// --8<-- [end:quickstart_add_feature]
// --8<-- [start:quickstart_query_feature]
let features: DataFrame = table
.query()
.select(Select::Columns(vec![
"name".to_string(),
"role".to_string(),
"power_score".to_string(),
]))
.execute()
.await
.unwrap()
.into_polars()
.await
.unwrap();
println!("{features:?}");
// --8<-- [end:quickstart_query_feature]
assert!(features.column("power_score").is_ok());
// --8<-- [start:quickstart_multimodal_bytes]
use std::sync::Arc;
use arrow_array::{
BinaryArray, FixedSizeListArray, LargeStringArray, RecordBatch, RecordBatchIterator,
};
use arrow_schema::{DataType, Field, Schema};
let image_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../../web/static/assets/images/quickstart/sir-lancelot.jpg");
let image_bytes = std::fs::read(image_path).unwrap();
let image_schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::LargeUtf8, false),
Field::new("description", DataType::LargeUtf8, false),
Field::new("image", DataType::Binary, false),
Field::new(
"vector",
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4),
false,
),
]));
let image_vectors = [[0.9_f32, 0.1, 0.5, 0.4]];
let image_batch = RecordBatch::try_new(
image_schema.clone(),
vec![
Arc::new(LargeStringArray::from_iter_values(["lancelot"])),
Arc::new(LargeStringArray::from_iter_values([
"Portrait of Sir Lancelot",
])),
Arc::new(BinaryArray::from_iter_values([image_bytes.as_slice()])),
Arc::new(
FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
image_vectors
.iter()
.map(|vector| Some(vector.iter().copied().map(Some).collect::<Vec<_>>())),
4,
),
),
],
)
.unwrap();
let image_reader: Box<dyn arrow_array::RecordBatchReader + Send> = Box::new(
RecordBatchIterator::new(vec![Ok(image_batch)].into_iter(), image_schema),
);
let multimodal_table = db
.create_table("character_images", image_reader)
.mode(CreateTableMode::Overwrite)
.execute()
.await
.unwrap();
// --8<-- [end:quickstart_multimodal_bytes]
assert_eq!(multimodal_table.count_rows(None).await.unwrap(), 1);
}
docs/web-tests/rs/search.rs
+311
View File
@@ -0,0 +1,311 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
//! This example demonstrates vector search query options.
//!
//! Snippets from this example are used in the documentation on vector search.
use std::sync::Arc;
use arrow_array::types::Float32Type;
use arrow_array::{
Array, FixedSizeListArray, Int32Array, Int64Array, RecordBatch, RecordBatchIterator,
RecordBatchReader, UInt8Array,
};
use arrow_schema::{DataType, Field, Schema};
use futures::TryStreamExt;
use lancedb::connection::Connection;
use lancedb::index::vector::IvfFlatIndexBuilder;
use lancedb::index::Index;
use lancedb::query::{ExecutableQuery, QueryBase, Select};
use lancedb::{connect, DistanceType, Result, Table};
const DIM: usize = 128;
#[tokio::main]
async fn main() -> Result<()> {
if std::path::Path::new("data").exists() {
std::fs::remove_dir_all("data").unwrap();
}
let uri = "data/sample-lancedb";
let db = connect(uri).execute().await?;
let tbl = create_table(&db).await?;
tbl.create_index(&["vector"], Index::Auto).execute().await?;
configure_distance_metric(&tbl).await?;
exact_vs_approximate(&tbl).await?;
search_distance_range(&tbl).await?;
vector_search_prefilter(&tbl).await?;
vector_search_postfilter(&tbl).await?;
fast_search(&tbl).await?;
brute_force_search(&tbl).await?;
bypass_vector_index(&tbl).await?;
batch_search(&tbl).await?;
binary_search(&db).await?;
Ok(())
}
fn create_some_records() -> Result<Box<dyn RecordBatchReader + Send>> {
const TOTAL: usize = 1000;
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int32, false),
Field::new(
"vector",
DataType::FixedSizeList(
Arc::new(Field::new("item", DataType::Float32, true)),
DIM as i32,
),
true,
),
]));
let batches = RecordBatchIterator::new(
vec![RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Int32Array::from_iter_values(0..TOTAL as i32)),
Arc::new(
FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
(0..TOTAL).map(|_| Some(vec![Some(1.0); DIM])),
DIM as i32,
),
),
],
)
.unwrap()]
.into_iter()
.map(Ok),
schema.clone(),
);
Ok(Box::new(batches))
}
async fn create_table(db: &Connection) -> Result<Table> {
let initial_data: Box<dyn RecordBatchReader + Send> = create_some_records()?;
let tbl = db
.create_table("my_vectors", initial_data)
.execute()
.await
.unwrap();
Ok(tbl)
}
async fn configure_distance_metric(table: &Table) -> Result<()> {
let query_vector = [1.0; DIM];
// --8<-- [start:configure_distance_metric]
// Use the same distance metric the index was trained with.
let mut results = table
.vector_search(&query_vector)?
.distance_type(DistanceType::Cosine)
.limit(10)
.execute()
.await?;
while let Some(batch) = results.try_next().await? {
println!("{:?}", batch);
}
// --8<-- [end:configure_distance_metric]
Ok(())
}
async fn exact_vs_approximate(table: &Table) -> Result<()> {
let query_vector = [1.0; DIM];
// --8<-- [start:exact_vs_approximate]
// Approximate ANN search (fast, distances may come from the index representation)
let mut fast_results = table.vector_search(&query_vector)?.limit(10).execute().await?;
while let Some(batch) = fast_results.try_next().await? {
println!("{:?}", batch);
}
// Rerank a larger candidate set on full vectors for better recall
let mut refined_results = table
.vector_search(&query_vector)?
.limit(10)
.refine_factor(20)
.execute()
.await?;
while let Some(batch) = refined_results.try_next().await? {
println!("{:?}", batch);
}
// --8<-- [end:exact_vs_approximate]
Ok(())
}
async fn search_distance_range(table: &Table) -> Result<()> {
let query_vector = [1.0; DIM];
// --8<-- [start:search_distance_range]
// Only return rows whose distance falls within [0.1, 0.5).
let mut results = table
.vector_search(&query_vector)?
.distance_range(Some(0.1), Some(0.5))
.execute()
.await?;
while let Some(batch) = results.try_next().await? {
println!("{:?}", batch);
}
// --8<-- [end:search_distance_range]
Ok(())
}
async fn vector_search_prefilter(table: &Table) -> Result<()> {
let query_vector = [1.0; DIM];
// --8<-- [start:vector_search_prefilter]
// Prefiltering is the default: the filter is applied before vector search.
let mut results = table
.vector_search(&query_vector)?
.only_if("id > 100")
.select(Select::columns(&["id"]))
.limit(5)
.execute()
.await?;
while let Some(batch) = results.try_next().await? {
println!("{:?}", batch);
}
// --8<-- [end:vector_search_prefilter]
Ok(())
}
async fn vector_search_postfilter(table: &Table) -> Result<()> {
let query_vector = [1.0; DIM];
// --8<-- [start:vector_search_postfilter]
// Apply the filter after vector search by calling postfilter().
let mut results = table
.vector_search(&query_vector)?
.only_if("id > 100")
.postfilter()
.select(Select::columns(&["id"]))
.limit(5)
.execute()
.await?;
while let Some(batch) = results.try_next().await? {
println!("{:?}", batch);
}
// --8<-- [end:vector_search_postfilter]
Ok(())
}
async fn fast_search(table: &Table) -> Result<()> {
let query_vector = [1.0; DIM];
// --8<-- [start:fast_search]
// Skip unindexed data for lower latency.
let mut results = table
.vector_search(&query_vector)?
.fast_search()
.limit(5)
.execute()
.await?;
while let Some(batch) = results.try_next().await? {
println!("{:?}", batch);
}
// --8<-- [end:fast_search]
Ok(())
}
async fn brute_force_search(table: &Table) -> Result<()> {
let query_vector = [1.0; DIM];
// --8<-- [start:brute_force_search]
// A plain vector search returns the top-k closest rows.
let mut results = table.vector_search(&query_vector)?.limit(3).execute().await?;
while let Some(batch) = results.try_next().await? {
println!("{:?}", batch);
}
// --8<-- [end:brute_force_search]
Ok(())
}
async fn bypass_vector_index(table: &Table) -> Result<()> {
let query_vector = [1.0; DIM];
// --8<-- [start:bypass_vector_index]
// Force an exhaustive (flat) scan for exact, ground-truth results.
let mut results = table
.vector_search(&query_vector)?
.bypass_vector_index()
.limit(5)
.execute()
.await?;
while let Some(batch) = results.try_next().await? {
println!("{:?}", batch);
}
// --8<-- [end:bypass_vector_index]
Ok(())
}
async fn batch_search(table: &Table) -> Result<()> {
let query_1 = [1.0; DIM];
let query_2 = [0.5; DIM];
// --8<-- [start:batch_search]
// Search multiple query vectors in one call. Each result row carries a
// `query_index` mapping it back to the query it matched.
let mut results = table
.vector_search(&query_1)?
.add_query_vector(&query_2)?
.limit(5)
.execute()
.await?;
while let Some(batch) = results.try_next().await? {
println!("{:?}", batch);
}
// --8<-- [end:batch_search]
Ok(())
}
async fn binary_search(db: &Connection) -> Result<()> {
// A 256-bit binary vector is stored as 256 / 8 = 32 packed uint8 bytes.
const NUM_BYTES: i32 = 32;
const TOTAL: usize = 1024;
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int64, false),
Field::new(
"vector",
DataType::FixedSizeList(
Arc::new(Field::new("item", DataType::UInt8, true)),
NUM_BYTES,
),
true,
),
]));
let values =
UInt8Array::from_iter_values((0..TOTAL * NUM_BYTES as usize).map(|i| (i % 256) as u8));
let vectors = FixedSizeListArray::try_new(
Arc::new(Field::new("item", DataType::UInt8, true)),
NUM_BYTES,
Arc::new(values),
None,
)
.unwrap();
let batch = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Int64Array::from_iter_values(0..TOTAL as i64)),
Arc::new(vectors),
],
)
.unwrap();
let reader: Box<dyn RecordBatchReader + Send> =
Box::new(RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema.clone()));
let tbl = db.create_table("binary_vectors", reader).execute().await?;
tbl.create_index(
&["vector"],
Index::IvfFlat(IvfFlatIndexBuilder::default().distance_type(DistanceType::Hamming)),
)
.execute()
.await?;
// --8<-- [start:binary_search]
// Binary vectors use `hamming` distance over the packed uint8 bytes.
let query: Arc<dyn Array> = Arc::new(UInt8Array::from(vec![1u8; NUM_BYTES as usize]));
let mut results = tbl
.vector_search(query)?
.distance_type(DistanceType::Hamming)
.limit(10)
.execute()
.await?;
while let Some(batch) = results.try_next().await? {
println!("{:?}", batch);
}
// --8<-- [end:binary_search]
Ok(())
}
docs/web-tests/rs/sentence_transformers.rs
+95
View File
@@ -0,0 +1,95 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::{iter::once, sync::Arc};
use arrow_array::{RecordBatch, RecordBatchIterator, StringArray};
use arrow_schema::{DataType, Field, Schema};
use futures::StreamExt;
use lancedb::{
arrow::IntoArrow,
connect,
embeddings::{
sentence_transformers::SentenceTransformersEmbeddings, EmbeddingDefinition,
EmbeddingFunction,
},
query::{ExecutableQuery, QueryBase},
Result,
};
#[tokio::main]
async fn main() -> Result<()> {
let tempdir = tempfile::tempdir().unwrap();
let tempdir = tempdir.path().to_str().unwrap();
let embedding = SentenceTransformersEmbeddings::builder().build()?;
let embedding = Arc::new(embedding);
let db = connect(tempdir).execute().await?;
db.embedding_registry()
.register("sentence-transformers", embedding.clone())?;
let table = db
.create_table("vectors", make_data())
.add_embedding(EmbeddingDefinition::new(
"facts",
"sentence-transformers",
Some("embeddings"),
))?
.execute()
.await?;
let query = Arc::new(StringArray::from_iter_values(once(
"How many bones are in the human body?",
)));
let query_vector = embedding.compute_query_embeddings(query)?;
let mut results = table
.vector_search(query_vector)?
.limit(1)
.execute()
.await?;
let rb = results.next().await.unwrap()?;
let out = rb
.column_by_name("facts")
.unwrap()
.as_any()
.downcast_ref::<StringArray>()
.unwrap();
let text = out.iter().next().unwrap().unwrap();
println!("Answer: {}", text);
Ok(())
}
fn make_data() -> impl IntoArrow {
let schema = Schema::new(vec![Field::new("facts", DataType::Utf8, false)]);
let facts = StringArray::from_iter_values(vec![
"Albert Einstein was a theoretical physicist.",
"The capital of France is Paris.",
"The Great Wall of China is one of the Seven Wonders of the World.",
"Python is a popular programming language.",
"Mount Everest is the highest mountain in the world.",
"Leonardo da Vinci painted the Mona Lisa.",
"Shakespeare wrote Hamlet.",
"The human body has 206 bones.",
"The speed of light is approximately 299,792 kilometers per second.",
"Water boils at 100 degrees Celsius.",
"The Earth orbits the Sun.",
"The Pyramids of Giza are located in Egypt.",
"Coffee is one of the most popular beverages in the world.",
"Tokyo is the capital city of Japan.",
"Photosynthesis is the process by which plants make their food.",
"The Pacific Ocean is the largest ocean on Earth.",
"Mozart was a prolific composer of classical music.",
"The Internet is a global network of computers.",
"Basketball is a sport played with a ball and a hoop.",
"The first computer virus was created in 1983.",
"Artificial neural networks are inspired by the human brain.",
"Deep learning is a subset of machine learning.",
"IBM's Watson won Jeopardy! in 2011.",
"The first computer programmer was Ada Lovelace.",
"The first chatbot was ELIZA, created in the 1960s.",
]);
let schema = Arc::new(schema);
let rb = RecordBatch::try_new(schema.clone(), vec![Arc::new(facts)]).unwrap();
Box::new(RecordBatchIterator::new(vec![Ok(rb)], schema))
}
docs/web-tests/rs/simple.rs
+147
View File
@@ -0,0 +1,147 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
//! This example demonstrates basic usage of LanceDb.
//!
//! Snippets from this example are used in the quickstart documentation.
use std::sync::Arc;
use arrow_array::types::Float32Type;
use arrow_array::{FixedSizeListArray, Int32Array, RecordBatch, RecordBatchIterator};
use arrow_schema::{DataType, Field, Schema};
use futures::TryStreamExt;
use lancedb::arrow::IntoArrow;
use lancedb::connection::Connection;
use lancedb::index::Index;
use lancedb::query::{ExecutableQuery, QueryBase};
use lancedb::{connect, Result, Table as LanceDbTable};
// --8<-- [start:connect]
#[tokio::main]
async fn main() -> Result<()> {
if std::path::Path::new("data").exists() {
std::fs::remove_dir_all("data").unwrap();
}
// --8<-- [start:connect_uri]
let uri = "data/sample-lancedb";
let db = connect(uri).execute().await?;
// --8<-- [end:connect_uri]
// --8<-- [start:list_names]
println!("{:?}", db.table_names().execute().await?);
// --8<-- [end:list_names]
let tbl = create_table(&db).await?;
create_index(&tbl).await?;
let batches = search(&tbl).await?;
println!("{:?}", batches);
create_empty_table(&db).await.unwrap();
// --8<-- [start:delete]
tbl.delete("id > 24").await.unwrap();
// --8<-- [end:delete]
// --8<-- [start:drop_table]
db.drop_table("my_table").await.unwrap();
// --8<-- [end:drop_table]
Ok(())
}
// --8<-- [end:connect]
#[allow(dead_code)]
async fn open_with_existing_tbl() -> Result<()> {
let uri = "data/sample-lancedb";
let db = connect(uri).execute().await?;
#[allow(unused_variables)]
// --8<-- [start:open_existing_tbl]
let table = db.open_table("my_table").execute().await.unwrap();
// --8<-- [end:open_existing_tbl]
Ok(())
}
fn create_some_records() -> Result<impl IntoArrow> {
const TOTAL: usize = 1000;
const DIM: usize = 128;
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int32, false),
Field::new(
"vector",
DataType::FixedSizeList(
Arc::new(Field::new("item", DataType::Float32, true)),
DIM as i32,
),
true,
),
]));
// Create a RecordBatch stream.
let batches = RecordBatchIterator::new(
vec![RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Int32Array::from_iter_values(0..TOTAL as i32)),
Arc::new(
FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
(0..TOTAL).map(|_| Some(vec![Some(1.0); DIM])),
DIM as i32,
),
),
],
)
.unwrap()]
.into_iter()
.map(Ok),
schema.clone(),
);
Ok(Box::new(batches))
}
async fn create_table(db: &Connection) -> Result<LanceDbTable> {
// --8<-- [start:create_table]
let initial_data = create_some_records()?;
let tbl = db
.create_table("my_table", initial_data)
.execute()
.await
.unwrap();
// --8<-- [end:create_table]
// --8<-- [start:add]
let new_data = create_some_records()?;
tbl.add(new_data).execute().await.unwrap();
// --8<-- [end:add]
Ok(tbl)
}
async fn create_empty_table(db: &Connection) -> Result<LanceDbTable> {
// --8<-- [start:create_empty_table]
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int32, false),
Field::new("item", DataType::Utf8, true),
]));
db.create_empty_table("empty_table", schema).execute().await
// --8<-- [end:create_empty_table]
}
async fn create_index(table: &LanceDbTable) -> Result<()> {
// --8<-- [start:create_index]
table.create_index(&["vector"], Index::Auto).execute().await
// --8<-- [end:create_index]
}
async fn search(table: &LanceDbTable) -> Result<Vec<RecordBatch>> {
// --8<-- [start:search]
table
.query()
.limit(2)
.nearest_to(&[1.0; 128])?
.execute()
.await?
.try_collect::<Vec<_>>()
.await
// --8<-- [end:search]
}
docs/web-tests/rs/tables.rs
+1516
View File
File diff suppressed because it is too large Load Diff
docs/web-tests/ts/ann_indexes.test.ts
+57
View File
@@ -0,0 +1,57 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { expect, test } from "@jest/globals";
// --8<-- [start:import]
import * as lancedb from "@lancedb/lancedb";
import type { VectorQuery } from "@lancedb/lancedb";
// --8<-- [end:import]
import { withTempDirectory } from "./util.ts";
test("ann index examples", async () => {
await withTempDirectory(async (databaseDir) => {
// --8<-- [start:ingest]
const db = await lancedb.connect(databaseDir);
const data = Array.from({ length: 5_000 }, (_, i) => ({
vector: Array(128).fill(i),
id: `${i}`,
content: "",
longId: `${i}`,
}));
const table = await db.createTable("my_vectors", data, {
mode: "overwrite",
});
await table.createIndex("vector", {
config: lancedb.Index.ivfPq({
numPartitions: 10,
numSubVectors: 16,
}),
});
// --8<-- [end:ingest]
// --8<-- [start:search1]
const search = table.search(Array(128).fill(1.2)).limit(2) as VectorQuery;
const results1 = await search.nprobes(20).refineFactor(10).toArray();
// --8<-- [end:search1]
expect(results1.length).toBe(2);
// --8<-- [start:search2]
const results2 = await table
.search(Array(128).fill(1.2))
.where("id != '1141'")
.limit(2)
.toArray();
// --8<-- [end:search2]
expect(results2.length).toBe(2);
// --8<-- [start:search3]
const results3 = await table
.search(Array(128).fill(1.2))
.select(["id"])
.limit(2)
.toArray();
// --8<-- [end:search3]
expect(results3.length).toBe(2);
});
}, 100_000);
docs/web-tests/ts/basic_usage.test.ts
+161
View File
@@ -0,0 +1,161 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { expect, test } from "@jest/globals";
import * as fs from "node:fs";
import * as path from "node:path";
// --8<-- [start:basic_imports]
import * as lancedb from "@lancedb/lancedb";
import * as arrow from "apache-arrow";
// --8<-- [end:basic_imports]
import { withTempDirectory } from "./util.ts";
const dataPath = path.join(__dirname, "..", "camelot.json");
test("basic usage examples (async)", async () => {
await withTempDirectory(async (databaseDir) => {
const db = await lancedb.connect(databaseDir);
// --8<-- [start:data_load]
const data = JSON.parse(fs.readFileSync(dataPath, "utf-8"));
// --8<-- [end:data_load]
// --8<-- [start:basic_create_table]
let table = await db.createTable("camelot", data, {
mode: "overwrite",
});
// --8<-- [end:basic_create_table]
expect(await table.countRows()).toBe(8);
// --8<-- [start:basic_open_table]
table = await db.openTable("camelot");
// --8<-- [end:basic_open_table]
// --8<-- [start:basic_create_empty_table]
const schema = new arrow.Schema([
new arrow.Field("id", new arrow.Int16()),
new arrow.Field("name", new arrow.Utf8()),
new arrow.Field("role", new arrow.Utf8()),
new arrow.Field("description", new arrow.Utf8()),
new arrow.Field(
"vector",
new arrow.FixedSizeList(
4,
new arrow.Field("item", new arrow.Float32(), true),
),
),
new arrow.Field(
"stats",
new arrow.Struct([
new arrow.Field("strength", new arrow.Int8()),
new arrow.Field("courage", new arrow.Int8()),
new arrow.Field("magic", new arrow.Int8()),
new arrow.Field("wisdom", new arrow.Int8()),
]),
),
]);
await db.createEmptyTable("camelot_empty", schema, { mode: "overwrite" });
// --8<-- [end:basic_create_empty_table]
expect(await db.tableNames()).toContain("camelot_empty");
await db.dropTable("camelot_empty");
// --8<-- [start:basic_add_data]
const magicalCharacters = [
{
id: 9,
name: "Morgan le Fay",
role: "Sorceress",
description:
"A powerful enchantress, Arthur's half-sister, and a complex figure who oscillates between aiding and opposing Camelot.",
vector: [0.1, 0.84, 0.25, 0.7],
stats: { strength: 2, courage: 3, magic: 5, wisdom: 4 },
},
{
id: 10,
name: "The Lady of the Lake",
role: "Mystical Guardian",
description:
"A mysterious supernatural figure associated with Avalon, known for giving Arthur the sword Excalibur.",
vector: [0.0, 0.9, 0.58, 0.88],
stats: { strength: 2, courage: 3, magic: 5, wisdom: 5 },
},
];
await table.add(magicalCharacters);
// --8<-- [end:basic_add_data]
expect(await table.countRows()).toBe(10);
// --8<-- [start:basic_vector_search]
const queryVector = [0.03, 0.85, 0.61, 0.9];
const result = await table.search(queryVector).limit(5).toArray();
console.log(result);
// --8<-- [end:basic_vector_search]
// --8<-- [start:basic_add_columns]
await table.addColumns([
{
name: "power",
valueSql:
"cast(((stats.strength + stats.courage + stats.magic + stats.wisdom) / 4.0) as float)",
},
]);
// --8<-- [end:basic_add_columns]
const schemaWithPower = await table.schema();
expect(schemaWithPower.fields.some((f) => f.name === "power")).toBe(true);
// --8<-- [start:basic_vector_search_q1]
// Who are the characters similar to "wizard"?
const queryVector1 = [0.03, 0.85, 0.61, 0.9];
const r1 = await table
.search(queryVector1)
.limit(5)
.select(["name", "role", "description"])
.toArray();
console.log(r1);
// --8<-- [end:basic_vector_search_q1]
// --8<-- [start:basic_vector_search_q2]
// Who are the characters similar to "wizard" with high magic stats?
const queryVector2 = [0.03, 0.85, 0.61, 0.9];
const r2 = await table
.search(queryVector2)
.where("stats.magic > 3")
.select(["name", "role", "description"])
.limit(5)
.toArray();
console.log(r2);
// --8<-- [end:basic_vector_search_q2]
// --8<-- [start:basic_vector_search_q3]
// Who are the strongest characters?
const r3 = await table
.query()
.where("stats.strength > 3")
.select(["name", "role", "description"])
.limit(5)
.toArray();
console.log(r3);
// --8<-- [end:basic_vector_search_q3]
// --8<-- [start:basic_vector_search_q4]
// Who are the strongest characters?
const r4 = await table
.query()
.select(["name", "role", "description", "power"])
.toArray();
console.log(r4);
// --8<-- [end:basic_vector_search_q4]
// --8<-- [start:basic_drop_columns]
await table.dropColumns(["power"]);
// --8<-- [end:basic_drop_columns]
// --8<-- [start:basic_delete_rows]
await table.delete('role = "Traitor Knight"');
// --8<-- [end:basic_delete_rows]
expect(await table.countRows()).toBe(9);
// --8<-- [start:basic_drop_table]
await db.dropTable("camelot");
// --8<-- [end:basic_drop_table]
expect(await db.tableNames()).not.toContain("camelot");
});
});
docs/web-tests/ts/biome.json
+52
View File
@@ -0,0 +1,52 @@
{
"$schema": "https://biomejs.dev/schemas/1.9.4/schema.json",
"vcs": {
"enabled": false,
"clientKind": "git",
"useIgnoreFile": false
},
"files": {
"ignoreUnknown": false,
"ignore": []
},
"formatter": {
"enabled": true,
"indentStyle": "space"
},
"organizeImports": {
"enabled": true
},
"linter": {
"enabled": true,
"rules": {
"recommended": true
}
},
"javascript": {
"formatter": {
"quoteStyle": "double"
}
},
"overrides": [
{
"include": ["*"],
"linter": {
"rules": {
"style": {
"noNonNullAssertion": "off"
}
}
}
},
{
"include": ["merge_insert.test.ts"],
"linter": {
"rules": {
"style": {
"useNamingConvention": "off"
}
}
}
}
]
}
docs/web-tests/ts/connection.test.ts
+121
View File
@@ -0,0 +1,121 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { expect, jest, test } from "@jest/globals";
import * as path from "node:path";
import { withTempDirectory } from "./util.ts";
// --8<-- [start:connect]
import * as lancedb from "@lancedb/lancedb";
async function connectExample(uri: string) {
const db = await lancedb.connect(uri);
return db;
}
// --8<-- [end:connect]
test("connect to a local database", async () => {
await withTempDirectory(async (tempDir) => {
const uri = path.join(tempDir, "ex_lancedb");
const db = await connectExample(uri);
expect(db).toBeDefined();
});
});
async function connectEnterpriseQuickstart() {
// --8<-- [start:connect_enterprise_quickstart]
const uri = "db://your-database-uri";
const apiKey = "your-api-key";
const region = "us-east-1";
const hostOverride = "https://your-enterprise-endpoint.com";
const db = await lancedb.connect(uri, {
apiKey,
region,
hostOverride,
});
// --8<-- [end:connect_enterprise_quickstart]
return db;
}
test("enterprise quickstart connect uses placeholder config", async () => {
const mockDb = { __mock: true } as unknown as Awaited<
ReturnType<typeof lancedb.connect>
>;
const spy = jest.spyOn(lancedb, "connect").mockResolvedValue(mockDb);
const db = await connectEnterpriseQuickstart();
expect(db).toBe(mockDb);
expect(spy).toHaveBeenCalledWith("db://your-database-uri", {
apiKey: "your-api-key",
region: "us-east-1",
hostOverride: "https://your-enterprise-endpoint.com",
});
spy.mockRestore();
});
// --8<-- [start:connect_object_storage]
async function connectObjectStorageExample() {
const uri = "s3://your-bucket/path";
// You can also use "gs://your-bucket/path" or "az://your-container/path".
const db = await lancedb.connect(uri);
return db;
}
// --8<-- [end:connect_object_storage]
async function namespaceTableOpsExample() {
// --8<-- [start:namespace_table_ops]
const db = await lancedb.connectNamespace("dir", { root: "./local_lancedb" });
// Create namespace tree: prod/search and prod/recommendations
await db.createNamespace(["prod"], { mode: "exist_ok" });
await db.createNamespace(["prod", "search"], { mode: "exist_ok" });
await db.createNamespace(["prod", "recommendations"], { mode: "exist_ok" });
await db.createTable(
"user",
[{ id: 1, vector: [0.1, 0.2], name: "alice" }],
["prod", "search"],
{ mode: "create" }, // use "overwrite" only if you want to replace existing table
);
await db.createTable(
"user",
[{ id: 2, vector: [0.3, 0.4], name: "bob" }],
["prod", "recommendations"],
{ mode: "create" },
);
// Verify
console.log((await db.listNamespaces()).namespaces); // ["prod"]
console.log((await db.listNamespaces(["prod"])).namespaces); // ["recommendations", "search"]
console.log(await db.tableNames(["prod", "search"])); // ["user"]
console.log(await db.tableNames(["prod", "recommendations"])); // ["user"]
// --8<-- [end:namespace_table_ops]
}
async function namespaceAdminOpsExample() {
// --8<-- [start:namespace_admin_ops]
const db = await lancedb.connectNamespace("dir", { root: "./local_lancedb" });
const namespace = ["prod", "search"];
await db.createNamespace(["prod"]);
await db.createNamespace(["prod", "search"]);
const childNamespaces = (await db.listNamespaces(["prod"])).namespaces;
console.log(`Child namespaces under ${JSON.stringify(namespace)}:`, childNamespaces);
// Child namespaces under ["prod","search"]: [ 'search' ]
const metadata = await db.describeNamespace(["prod", "search"]);
console.log(`Metadata for namespace ${JSON.stringify(namespace)}:`, metadata);
await db.dropNamespace(["prod", "search"], { mode: "skip" });
await db.dropNamespace(["prod"], { mode: "skip" });
// --8<-- [end:namespace_admin_ops]
return { childNamespaces, metadata };
}
void [
connectObjectStorageExample,
connectEnterpriseQuickstart,
namespaceTableOpsExample,
namespaceAdminOpsExample,
];
docs/web-tests/ts/custom_embedding_function.test.ts
+79
View File
@@ -0,0 +1,79 @@
import {
type FeatureExtractionPipeline,
pipeline,
} from "@huggingface/transformers";
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { expect, test } from "@jest/globals";
// --8<-- [start:imports]
import * as lancedb from "@lancedb/lancedb";
import {
LanceSchema,
TextEmbeddingFunction,
getRegistry,
register,
} from "@lancedb/lancedb/embedding";
// --8<-- [end:imports]
import { withTempDirectory } from "./util.ts";
// --8<-- [start:embedding_impl]
@register("sentence-transformers")
class SentenceTransformersEmbeddings extends TextEmbeddingFunction {
name = "Xenova/all-miniLM-L6-v2";
#ndims!: number;
extractor!: FeatureExtractionPipeline;
async init() {
this.extractor = await pipeline("feature-extraction", this.name, {
dtype: "fp32",
});
this.#ndims = await this.generateEmbeddings(["hello"]).then(
(e) => e[0].length,
);
}
ndims() {
return this.#ndims;
}
toJSON() {
return {
name: this.name,
};
}
async generateEmbeddings(texts: string[]) {
const output = await this.extractor(texts, {
pooling: "mean",
normalize: true,
});
return output.tolist();
}
}
// --8<-- [end:embedding_impl]
test("Registry examples", async () => {
await withTempDirectory(async (databaseDir) => {
// --8<-- [start:call_custom_function]
const registry = getRegistry();
const sentenceTransformer = await registry
.get<SentenceTransformersEmbeddings>("sentence-transformers")!
.create();
const schema = LanceSchema({
vector: sentenceTransformer.vectorField(),
text: sentenceTransformer.sourceField(),
});
const db = await lancedb.connect(databaseDir);
const table = await db.createEmptyTable("table", schema, {
mode: "overwrite",
});
await table.add([{ text: "hello" }, { text: "world" }]);
const results = await table.search("greeting").limit(1).toArray();
// --8<-- [end:call_custom_function]
expect(results.length).toBe(1);
});
}, 100_000);
docs/web-tests/ts/embedding.test.ts
+160
View File
@@ -0,0 +1,160 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { expect, test } from "@jest/globals";
// --8<-- [start:imports]
import * as lancedb from "@lancedb/lancedb";
import "@lancedb/lancedb/embedding/openai";
import { LanceSchema, getRegistry, register } from "@lancedb/lancedb/embedding";
import { EmbeddingFunction } from "@lancedb/lancedb/embedding";
import { type Float, Float32, Utf8 } from "apache-arrow";
// --8<-- [end:imports]
import { withTempDirectory } from "./util.ts";
const openAiTest = process.env.OPENAI_API_KEY == null ? test.skip : test;
openAiTest("create embedding function", () => {
// --8<-- [start:create_embedding_function]
const func = getRegistry().get("openai")!.create({
model: "text-embedding-3-small",
});
// --8<-- [end:create_embedding_function]
expect(func).toBeDefined();
});
openAiTest("openai embeddings", async () => {
await withTempDirectory(async (databaseDir) => {
// --8<-- [start:openai_embeddings]
const db = await lancedb.connect(databaseDir);
const func = getRegistry()
.get("openai")
?.create({ model: "text-embedding-ada-002" }) as EmbeddingFunction;
const wordsSchema = LanceSchema({
text: func.sourceField(new Utf8()),
vector: func.vectorField(),
});
const tbl = await db.createEmptyTable("words", wordsSchema, {
mode: "overwrite",
});
await tbl.add([{ text: "hello world" }, { text: "goodbye world" }]);
const query = "greetings";
const actual = (await tbl.search(query).limit(1).toArray())[0];
// --8<-- [end:openai_embeddings]
expect(actual).toHaveProperty("text");
});
});
openAiTest("manual query embeddings", async () => {
await withTempDirectory(async (databaseDir) => {
// --8<-- [start:manual_query_embeddings]
const db = await lancedb.connect(databaseDir);
const func = getRegistry()
.get("openai")
?.create({ model: "text-embedding-ada-002" }) as EmbeddingFunction;
const wordsSchema = LanceSchema({
text: func.sourceField(new Utf8()),
vector: func.vectorField(),
});
const tbl = await db.createEmptyTable("words", wordsSchema, {
mode: "overwrite",
});
await tbl.add([{ text: "hello world" }, { text: "goodbye world" }]);
const queryVector = await func.computeQueryEmbeddings("greetings");
// --8<-- [start:manual_query_search]
// queryVector is assumed to already be generated by your embedding function
const actual = (await tbl.search(queryVector).limit(1).toArray())[0];
// --8<-- [end:manual_query_search]
// --8<-- [end:manual_query_embeddings]
expect(actual).toHaveProperty("text");
});
});
test("custom embedding function", async () => {
await withTempDirectory(async (databaseDir) => {
// --8<-- [start:embedding_function]
const db = await lancedb.connect(databaseDir);
@register("my_embedding")
class MyEmbeddingFunction extends EmbeddingFunction<string> {
constructor(optionsRaw = {}) {
super();
const options = this.resolveVariables(optionsRaw);
// Initialize using options
}
ndims() {
return 3;
}
protected getSensitiveKeys(): string[] {
return [];
}
embeddingDataType(): Float {
return new Float32();
}
async computeQueryEmbeddings(_data: string) {
// This is a placeholder for a real embedding function
return [1, 2, 3];
}
async computeSourceEmbeddings(data: string[]) {
// This is a placeholder for a real embedding function
return Array.from({ length: data.length }).fill([
1, 2, 3,
]) as number[][];
}
}
const func = new MyEmbeddingFunction();
const data = [{ text: "pepperoni" }, { text: "pineapple" }];
// Option 1: manually specify the embedding function
const table = await db.createTable("vectors", data, {
embeddingFunction: {
function: func,
sourceColumn: "text",
vectorColumn: "vector",
},
mode: "overwrite",
});
// Option 2: provide the embedding function through a schema
const schema = LanceSchema({
text: func.sourceField(new Utf8()),
vector: func.vectorField(),
});
const table2 = await db.createTable("vectors2", data, {
schema,
mode: "overwrite",
});
// --8<-- [end:embedding_function]
expect(await table.countRows()).toBe(2);
expect(await table2.countRows()).toBe(2);
});
});
test("embedding function api_key", async () => {
// --8<-- [start:register_secret]
const registry = getRegistry();
registry.setVar("api_key", "sk-...");
const func = registry.get("openai")!.create({
apiKey: "$var:api_key",
});
// --8<-- [end:register_secret]
});
openAiTest("embedding function variable fallback", () => {
// --8<-- [start:register_model_fallback]
const registry = getRegistry();
registry.setVar("openai_model", "text-embedding-3-large");
const func = registry.get("openai")!.create({
model: "$var:openai_model:text-embedding-3-small",
});
// --8<-- [end:register_model_fallback]
expect(func).toBeDefined();
});
docs/web-tests/ts/filtering.test.ts
+42
View File
@@ -0,0 +1,42 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { expect, test } from "@jest/globals";
import * as lancedb from "@lancedb/lancedb";
import { withTempDirectory } from "./util.ts";
test("filtering examples", async () => {
await withTempDirectory(async (databaseDir) => {
const db = await lancedb.connect(databaseDir);
const data = Array.from({ length: 10_000 }, (_, i) => ({
vector: Array(1536).fill(i),
id: i,
item: `item ${i}`,
strId: `${i}`,
}));
const tbl = await db.createTable("myVectors", data, { mode: "overwrite" });
// --8<-- [start:search]
const _result = await tbl
.search(Array(1536).fill(0.5))
.limit(1)
.where("id = 10")
.toArray();
// --8<-- [end:search]
// --8<-- [start:vec_search]
const result = await (
tbl.search(Array(1536).fill(0)) as lancedb.VectorQuery
)
.where("(item IN ('item 0', 'item 2')) AND (id > 10)")
.postfilter()
.toArray();
// --8<-- [end:vec_search]
expect(result.length).toBe(0);
// --8<-- [start:sql_search]
await tbl.query().where("id = 10").limit(10).toArray();
// --8<-- [end:sql_search]
});
});
docs/web-tests/ts/full_text_search.test.ts
+45
View File
@@ -0,0 +1,45 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { expect, test } from "@jest/globals";
import * as lancedb from "@lancedb/lancedb";
import { withTempDirectory } from "./util.ts";
test("full text search", async () => {
await withTempDirectory(async (databaseDir) => {
const db = await lancedb.connect(databaseDir);
const words = [
"apple",
"banana",
"cherry",
"date",
"elderberry",
"fig",
"grape",
];
const data = Array.from({ length: 10_000 }, (_, i) => ({
vector: Array(1536).fill(i),
id: i,
item: `item ${i}`,
strId: `${i}`,
doc: words[i % words.length],
}));
const tbl = await db.createTable("myVectors", data, { mode: "overwrite" });
await tbl.createIndex("doc", {
config: lancedb.Index.fts(),
});
// --8<-- [start:full_text_search]
const result = await tbl
.query()
.nearestToText("apple")
.select(["id", "doc"])
.limit(10)
.toArray();
expect(result.length).toBe(10);
// --8<-- [end:full_text_search]
});
}, 10_000);
docs/web-tests/ts/integrations.ts
+134
View File
@@ -0,0 +1,134 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// --8<-- [start:frameworks_genkit_usage]
import { lancedbIndexerRef, lancedb, lancedbRetrieverRef, WriteMode } from "genkitx-lancedb";
import { textEmbedding004, vertexAI } from "@genkit-ai/vertexai";
import { gemini } from "@genkit-ai/vertexai";
import { z, genkit } from "genkit";
import { Document } from "genkit/retriever";
import { chunk } from "llm-chunk";
import { readFile } from "fs/promises";
import path from "path";
import pdf from "pdf-parse/lib/pdf-parse";
const ai = genkit({
plugins: [
// vertexAI provides the textEmbedding004 embedder
vertexAI(),
// the local vector store requires an embedder to translate from text to vector
lancedb([
{
dbUri: ".db", // optional lancedb uri, default to .db
tableName: "table", // optional table name, default to table
embedder: textEmbedding004,
},
]),
],
});
// --8<-- [end:frameworks_genkit_usage]
// --8<-- [start:frameworks_genkit_custom_indexer]
export const menuPdfIndexer = lancedbIndexerRef({
// Using all defaults, for dbUri, tableName, and embedder, etc
});
const chunkingConfig = {
minLength: 1000,
maxLength: 2000,
splitter: "sentence",
overlap: 100,
delimiters: "",
} as any;
async function extractTextFromPdf(filePath: string) {
const pdfFile = path.resolve(filePath);
const dataBuffer = await readFile(pdfFile);
const data = await pdf(dataBuffer);
return data.text;
}
export const indexMenu = ai.defineFlow(
{
name: "indexMenu",
inputSchema: z.string().describe("PDF file path"),
outputSchema: z.void(),
},
async (filePath: string) => {
filePath = path.resolve(filePath);
// Read the pdf.
const pdfTxt = await ai.run("extract-text", () => extractTextFromPdf(filePath));
// Divide the pdf text into segments.
const chunks = await ai.run("chunk-it", async () => chunk(pdfTxt, chunkingConfig));
// Convert chunks of text into documents to store in the index.
const documents = chunks.map((text) => {
return Document.fromText(text, { filePath });
});
// Add documents to the index.
await ai.index({
indexer: menuPdfIndexer,
documents,
options: {
writeMode: WriteMode.Overwrite,
} as any,
});
},
);
// --8<-- [end:frameworks_genkit_custom_indexer]
// --8<-- [start:frameworks_genkit_custom_retriever]
export const menuRetriever = lancedbRetrieverRef({
tableName: "table", // Use the same table name as the indexer.
displayName: "Menu", // Use a custom display name.
});
export const menuQAFlow = ai.defineFlow(
{ name: "Menu", inputSchema: z.string(), outputSchema: z.string() },
async (input: string) => {
// retrieve relevant documents
const docs = await ai.retrieve({
retriever: menuRetriever,
query: input,
options: {
k: 3,
},
});
const extractedContent = docs.map((doc) => {
if (doc.content && Array.isArray(doc.content) && doc.content.length > 0) {
if (doc.content[0].media && doc.content[0].media.url) {
return doc.content[0].media.url;
}
}
return "No content found";
});
console.log("Extracted content:", extractedContent);
const { text } = await ai.generate({
model: gemini("gemini-2.0-flash"),
prompt: `
You are acting as a helpful AI assistant that can answer
questions about the food available on the menu at Genkit Grub Pub.
Use only the context provided to answer the question.
If you don't know, do not make up an answer.
Do not add or change items on the menu.
Context:
${extractedContent.join("\n\n")}
Question: ${input}`,
docs,
});
return text;
},
);
// --8<-- [end:frameworks_genkit_custom_retriever]
docs/web-tests/ts/jest.config.cjs
+6
View File
@@ -0,0 +1,6 @@
/** @type {import('ts-jest').JestConfigWithTsJest} */
module.exports = {
preset: "ts-jest",
testEnvironment: "node",
testPathIgnorePatterns: ["./dist"],
};
docs/web-tests/ts/merge_insert.test.ts
+68
View File
@@ -0,0 +1,68 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { expect, test } from "@jest/globals";
import * as lancedb from "@lancedb/lancedb";
test("basic upsert", async () => {
const db = await lancedb.connect("memory://");
// --8<-- [start:upsert_basic]
const table = await db.createTable("users", [
{ id: 0, name: "Alice" },
{ id: 1, name: "Bob" },
]);
const newUsers = [
{ id: 1, name: "Bobby" },
{ id: 2, name: "Charlie" },
];
await table
.mergeInsert("id")
.whenMatchedUpdateAll()
.whenNotMatchedInsertAll()
.execute(newUsers);
await table.countRows(); // 3
// --8<-- [end:upsert_basic]
expect(await table.countRows()).toBe(3);
// --8<-- [start:insert_if_not_exists]
const table2 = await db.createTable("domains", [
{ domain: "google.com", name: "Google" },
{ domain: "github.com", name: "GitHub" },
]);
const newDomains = [
{ domain: "google.com", name: "Google" },
{ domain: "facebook.com", name: "Facebook" },
];
await table2
.mergeInsert("domain")
.whenNotMatchedInsertAll()
.execute(newDomains);
await table2.countRows(); // 3
// --8<-- [end:insert_if_not_exists]
expect(await table2.countRows()).toBe(3);
// --8<-- [start:replace_range]
const table3 = await db.createTable("chunks", [
{ doc_id: 0, chunk_id: 0, text: "Hello" },
{ doc_id: 0, chunk_id: 1, text: "World" },
{ doc_id: 1, chunk_id: 0, text: "Foo" },
{ doc_id: 1, chunk_id: 1, text: "Bar" },
]);
const newChunks = [{ doc_id: 1, chunk_id: 0, text: "Baz" }];
await table3
.mergeInsert(["doc_id", "chunk_id"])
.whenMatchedUpdateAll()
.whenNotMatchedInsertAll()
.whenNotMatchedBySourceDelete({ where: "doc_id = 1" })
.execute(newChunks);
await table3.countRows("doc_id = 1"); // 1
// --8<-- [end:replace_range]
expect(await table3.countRows("doc_id = 1")).toBe(1);
});
docs/web-tests/ts/multimodal.test.ts
+104
View File
@@ -0,0 +1,104 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { expect, test } from "@jest/globals";
// --8<-- [start:multimodal_imports]
import * as arrow from "apache-arrow";
import { Buffer } from "node:buffer";
import * as lancedb from "@lancedb/lancedb";
// --8<-- [end:multimodal_imports]
import { withTempDirectory } from "./util.ts";
test("multimodal snippets (async)", async () => {
await withTempDirectory(async (databaseDir) => {
const db = await lancedb.connect(databaseDir);
// --8<-- [start:create_dummy_data]
const createDummyImage = (color: string): Uint8Array => {
const pngHeader = Uint8Array.from([137, 80, 78, 71, 13, 10, 26, 10]);
return Buffer.concat([Buffer.from(pngHeader), Buffer.from(color, "utf8")]);
};
const data = [
{
id: 1,
filename: "red_square.png",
vector: Array.from({ length: 128 }, (_, i) => (i % 16) / 16),
image_blob: createDummyImage("red"),
label: "red",
},
{
id: 2,
filename: "blue_square.png",
vector: Array.from({ length: 128 }, (_, i) => ((i + 8) % 16) / 16),
image_blob: createDummyImage("blue"),
label: "blue",
},
];
// --8<-- [end:create_dummy_data]
// --8<-- [start:define_schema]
const schema = new arrow.Schema([
new arrow.Field("id", new arrow.Int32()),
new arrow.Field("filename", new arrow.Utf8()),
new arrow.Field(
"vector",
new arrow.FixedSizeList(
128,
new arrow.Field("item", new arrow.Float32(), true),
),
),
new arrow.Field("image_blob", new arrow.Binary()),
new arrow.Field("label", new arrow.Utf8()),
]);
// --8<-- [end:define_schema]
// --8<-- [start:ingest_data]
const multimodalData = lancedb.makeArrowTable(data, { schema });
const tbl = await db.createTable("images", multimodalData, {
mode: "overwrite",
});
// --8<-- [end:ingest_data]
expect(await tbl.countRows()).toBe(2);
// --8<-- [start:search_data]
const queryVector = Array.from({ length: 128 }, (_, i) => (i % 16) / 16);
const results = await tbl.search(queryVector).limit(1).toArray();
// --8<-- [end:search_data]
// --8<-- [start:process_results]
for (const row of results) {
const imageBytes = row.image_blob as Uint8Array;
console.log(
`Retrieved image: ${row.filename}, Byte length: ${imageBytes.length}`,
);
}
// --8<-- [end:process_results]
expect(results).toHaveLength(1);
// --8<-- [start:blob_api_schema]
const blobSchema = new arrow.Schema([
new arrow.Field("id", new arrow.Int64()),
new arrow.Field(
"video",
new arrow.LargeBinary(),
true,
new Map([["lance-encoding:blob", "true"]]),
),
]);
// --8<-- [end:blob_api_schema]
// --8<-- [start:blob_api_ingest]
const blobData = lancedb.makeArrowTable(
[
{ id: 1, video: Buffer.from("fake_video_bytes_1") },
{ id: 2, video: Buffer.from("fake_video_bytes_2") },
],
{ schema: blobSchema },
);
const blobTable = await db.createTable("videos", blobData, {
mode: "overwrite",
});
// --8<-- [end:blob_api_ingest]
expect(await blobTable.countRows()).toBe(2);
});
});
docs/web-tests/ts/package-lock.json
Generated
+5400
View File
File diff suppressed because it is too large Load Diff
docs/web-tests/ts/package.json
+36
View File
@@ -0,0 +1,36 @@
{
"name": "examples",
"version": "1.0.0",
"description": "Examples for LanceDB",
"main": "index.js",
"type": "module",
"scripts": {
"//1": "--experimental-vm-modules is needed to run jest with sentence-transformers",
"//2": "--testEnvironment is needed to run jest with sentence-transformers",
"//3": "See: https://github.com/huggingface/transformers.js/issues/57",
"test": "node --experimental-vm-modules node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose",
"lint": "biome check *.ts && biome format *.ts",
"lint-ci": "biome ci .",
"lint-fix": "biome check --write *.ts && npm run format",
"format": "biome format --write *.ts"
},
"author": "Lance Devs",
"license": "Apache-2.0",
"dependencies": {
"@huggingface/transformers": "^3.0.2",
"@lancedb/lancedb": "^0.31.0",
"openai": "^4.29.2",
"sharp": "^0.35.5"
},
"devDependencies": {
"@biomejs/biome": "^1.7.3",
"@jest/globals": "^29.7.0",
"jest": "^29.7.0",
"jest-environment-node-single-context": "^29.4.0",
"ts-jest": "^29.2.5",
"typescript": "^5.5.4"
},
"overrides": {
"sharp": "$sharp"
}
}
docs/web-tests/ts/quickstart.test.ts
+176
View File
@@ -0,0 +1,176 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { expect, test } from "@jest/globals";
import * as lancedb from "@lancedb/lancedb";
import { withTempDirectory } from "./util.ts";
test("quickstart example (async)", async () => {
await withTempDirectory(async (databaseDir) => {
const db = await lancedb.connect(databaseDir);
// --8<-- [start:quickstart_data]
const data = [
{
id: "1",
name: "King Arthur",
role: "King",
description: "Leader of Camelot and wielder of Excalibur.",
stats: { strength: 4, magic: 1, leadership: 5, wisdom: 4 },
vector: [0.7, 0.1, 0.9, 0.7],
},
{
id: "2",
name: "Merlin",
role: "Wizard",
description: "Advisor and prophet with deep magical knowledge.",
stats: { strength: 2, magic: 5, leadership: 4, wisdom: 5 },
vector: [0.2, 0.9, 0.4, 0.9],
},
{
id: "3",
name: "Sir Lancelot",
role: "Knight",
description: "Legendary knight known for courage and combat skill.",
stats: { strength: 5, magic: 1, leadership: 3, wisdom: 3 },
vector: [0.9, 0.1, 0.5, 0.4],
},
];
// --8<-- [end:quickstart_data]
// --8<-- [start:quickstart_create_table]
let table = await db.createTable("characters", data, { mode: "overwrite" });
// --8<-- [end:quickstart_create_table]
expect(await table.countRows()).toBe(3);
await db.dropTable("characters");
// --8<-- [start:quickstart_create_table_no_overwrite]
table = await db.createTable("characters", data);
// --8<-- [end:quickstart_create_table_no_overwrite]
expect(await table.countRows()).toBe(3);
// --8<-- [start:quickstart_vector_search_1]
// Search for examples similar to a "wise magical advisor"
let queryVector = [0.2, 0.8, 0.4, 0.9];
let result = await table
.search(queryVector)
.select(["name", "role", "description", "_distance"])
.limit(2)
.toArray();
console.table(result);
// --8<-- [end:quickstart_vector_search_1]
expect(result[0].name).toBe("Merlin");
// --8<-- [start:quickstart_curate_with_metadata]
const curated = await table
.search(queryVector)
.where("stats.magic >= 4")
.select(["name", "role", "description", "_distance"])
.limit(2)
.toArray();
console.table(curated);
// --8<-- [end:quickstart_curate_with_metadata]
expect(curated[0].name).toBe("Merlin");
// --8<-- [start:quickstart_output_array]
result = await table.search(queryVector).limit(2).toArray();
console.table(result);
// --8<-- [end:quickstart_output_array]
expect(result[0].name).toBe("Merlin");
// --8<-- [start:quickstart_add_feature]
await table.addColumns([
{
name: "power_score",
valueSql:
"cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)",
},
]);
// --8<-- [end:quickstart_add_feature]
const schemaWithFeature = await table.schema();
expect(schemaWithFeature.fields.some((f) => f.name === "power_score")).toBe(
true,
);
// --8<-- [start:quickstart_query_feature]
const features = await table
.query()
.select(["name", "role", "power_score"])
.toArray();
console.table(features);
// --8<-- [end:quickstart_query_feature]
expect(features[0]).toHaveProperty("power_score");
// --8<-- [start:quickstart_multimodal_bytes]
const arrow = await import("apache-arrow");
const path = await import("node:path");
const { readFile } = await import("node:fs/promises");
const imagePath = path.join(
__dirname,
"../../web/static/assets/images/quickstart/sir-lancelot.jpg",
);
const imageBytes = await readFile(imagePath);
const imageSchema = new arrow.Schema([
new arrow.Field("id", new arrow.Utf8()),
new arrow.Field("description", new arrow.Utf8()),
new arrow.Field("image", new arrow.Binary()),
new arrow.Field(
"vector",
new arrow.FixedSizeList(
4,
new arrow.Field("item", new arrow.Float32(), true),
),
),
]);
const imageData = lancedb.makeArrowTable(
[
{
id: "lancelot",
description: "Portrait of Sir Lancelot",
image: imageBytes,
vector: [0.9, 0.1, 0.5, 0.4],
},
],
{ schema: imageSchema },
);
const multimodalTable = await db.createTable(
"character_images",
imageData,
{ mode: "overwrite" },
);
// --8<-- [end:quickstart_multimodal_bytes]
expect(await multimodalTable.countRows()).toBe(1);
// --8<-- [start:quickstart_open_table]
table = await db.openTable("characters");
// --8<-- [end:quickstart_open_table]
// --8<-- [start:quickstart_add_data]
const moreData = [
{
id: "4",
name: "Morgana",
role: "Sorceress",
description: "Powerful sorceress of Avalon.",
stats: { strength: 2, magic: 5, leadership: 4, wisdom: 4 },
vector: [0.3, 0.9, 0.6, 0.8],
power_score: 3.75,
},
];
// Add data to table
await table.add(moreData);
// --8<-- [end:quickstart_add_data]
expect(await table.countRows()).toBe(4);
// --8<-- [start:quickstart_vector_search_2]
// Search for examples similar to a "powerful sorceress"
queryVector = [0.3, 0.9, 0.6, 0.8];
const results = await table.search(queryVector).limit(2).toArray();
console.table(results);
// --8<-- [end:quickstart_vector_search_2]
expect(results[0].name).toBe("Morgana");
});
});
docs/web-tests/ts/search.test.ts
+355
View File
@@ -0,0 +1,355 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { expect, test } from "@jest/globals";
// --8<-- [start:import]
import * as lancedb from "@lancedb/lancedb";
// --8<-- [end:import]
// --8<-- [start:import_bin_util]
import { Field, FixedSizeList, Int32, Schema, Uint8 } from "apache-arrow";
// --8<-- [end:import_bin_util]
import { Float32, Struct } from "apache-arrow";
import { withTempDirectory } from "./util.ts";
async function buildIndexedTable(db: lancedb.Connection, name: string) {
const data = Array.from({ length: 512 }, (_, i) => ({
vector: Array.from({ length: 128 }, () => Math.random()),
id: i,
}));
const tbl = await db.createTable(name, data, { mode: "overwrite" });
await tbl.createIndex("vector", {
config: lancedb.Index.ivfPq({ numPartitions: 4, numSubVectors: 8 }),
});
return tbl;
}
test("vector search", async () => {
await withTempDirectory(async (databaseDir) => {
{
const db = await lancedb.connect(databaseDir);
const data = Array.from({ length: 10_000 }, (_, i) => ({
vector: Array(128).fill(i),
id: `${i}`,
}));
await db.createTable("my_vectors", data);
}
// --8<-- [start:search1]
const db = await lancedb.connect(databaseDir);
const tbl = await db.openTable("my_vectors");
const results1 = await tbl.search(Array(128).fill(1.2)).limit(10).toArray();
// --8<-- [end:search1]
expect(results1.length).toBe(10);
// --8<-- [start:search2]
const results2 = await (
tbl.search(Array(128).fill(1.2)) as lancedb.VectorQuery
)
.distanceType("cosine")
.limit(10)
.toArray();
// --8<-- [end:search2]
expect(results2.length).toBe(10);
// --8<-- [start:distance_range]
const results3 = await (
tbl.search(Array(128).fill(1.2)) as lancedb.VectorQuery
)
.distanceType("cosine")
.distanceRange(0.1, 0.2)
.limit(10)
.toArray();
// --8<-- [end:distance_range]
for (const r of results3) {
expect(r.distance).toBeGreaterThanOrEqual(0.1);
expect(r.distance).toBeLessThan(0.2);
}
{
// --8<-- [start:ingest_binary_data]
const schema = new Schema([
new Field("id", new Int32(), true),
new Field("vec", new FixedSizeList(32, new Field("item", new Uint8()))),
]);
const data = lancedb.makeArrowTable(
Array(1_000)
.fill(0)
.map((_, i) => ({
// the 256 bits would be store in 32 bytes,
// if your data is already in this format, you can skip the packBits step
id: i,
vec: lancedb.packBits(Array(256).fill(i % 2)),
})),
{ schema: schema },
);
const tbl = await db.createTable("binary_table", data);
await tbl.createIndex("vec", {
config: lancedb.Index.ivfFlat({
numPartitions: 10,
distanceType: "hamming",
}),
});
// --8<-- [end:ingest_binary_data]
// --8<-- [start:search_binary_data]
const query = Array(32)
.fill(1)
.map(() => Math.floor(Math.random() * 255));
const results = await tbl.query().nearestTo(query).limit(10).toArrow();
// --8<-- [end:search_binary_data]
expect(results.numRows).toBe(10);
}
});
});
test("vector search docs snippets", async () => {
await withTempDirectory(async (databaseDir) => {
const db = await lancedb.connect(databaseDir);
// ---- Nested vector column: inference and explicit selection ----
{
const nestedSchema = new Schema([
new Field("id", new Int32()),
new Field(
"image",
new Struct([
new Field(
"embedding",
new FixedSizeList(2, new Field("item", new Float32(), true)),
),
]),
),
]);
const arrowTbl = lancedb.makeArrowTable(
[{ id: 0, image: { embedding: [0.0, 1.0] } }],
{ schema: nestedSchema },
);
await db.createTable("nested", arrowTbl, { mode: "overwrite" });
// --8<-- [start:select_vector_column]
const table = await db.openTable("nested");
// Inferred: LanceDB finds the single nested vector leaf automatically.
await table.query().nearestTo([0.0, 1.0]).limit(1).toArray();
// Explicit: required when more than one vector column matches.
await table
.query()
.nearestTo([0.0, 1.0])
.column("image.embedding")
.limit(1)
.toArray();
// --8<-- [end:select_vector_column]
}
// ---- Index a nested vector column ----
{
const dim = 16;
const nestedSchema = new Schema([
new Field("id", new Int32()),
new Field(
"image",
new Struct([
new Field(
"embedding",
new FixedSizeList(dim, new Field("item", new Float32(), true)),
),
]),
),
]);
const rows = Array.from({ length: 512 }, (_, i) => ({
id: i,
image: { embedding: Array.from({ length: dim }, () => Math.random()) },
}));
const arrowTbl = lancedb.makeArrowTable(rows, { schema: nestedSchema });
const table = await db.createTable("nested_index", arrowTbl, {
mode: "overwrite",
});
// --8<-- [start:index_nested_column]
await table.createIndex("image.embedding");
// --8<-- [end:index_nested_column]
}
// ---- Indexed queries: refine, fast search, bypass index ----
{
const table = await buildIndexedTable(db, "ann_table");
const embedding = Array.from({ length: 128 }, () => Math.random());
// --8<-- [start:exact_vs_approximate]
// Indexed ANN search without refinement (fast, approximate `_distance`)
const fastResults = await (table.search(embedding) as lancedb.VectorQuery)
.limit(10)
.toArray();
// Recompute distances on full vectors for reranked candidates
const exactDistanceResults = await (
table.search(embedding) as lancedb.VectorQuery
)
.limit(10)
.refineFactor(1)
.toArray();
// Rerank a larger candidate set for better recall (higher latency)
const higherRecallResults = await (
table.search(embedding) as lancedb.VectorQuery
)
.limit(10)
.refineFactor(20)
.toArray();
// --8<-- [end:exact_vs_approximate]
expect(fastResults.length).toBe(10);
expect(exactDistanceResults.length).toBe(10);
expect(higherRecallResults.length).toBe(10);
// --8<-- [start:fast_search]
await table
.query()
.nearestTo(embedding)
.fastSearch()
.limit(5)
.toArray();
// --8<-- [end:fast_search]
// --8<-- [start:bypass_vector_index]
await table
.query()
.nearestTo(embedding)
.bypassVectorIndex()
.limit(5)
.toArray();
// --8<-- [end:bypass_vector_index]
}
// ---- Brute force search (no index) ----
{
const data = Array.from({ length: 64 }, (_, i) => ({
vector: Array(128).fill(i),
id: `${i}`,
}));
await db.createTable("my_vectors", data, { mode: "overwrite" });
// --8<-- [start:brute_force_search]
const tbl = await db.openTable("my_vectors");
const results1 = await tbl.search(Array(128).fill(1.2)).limit(3).toArray();
// --8<-- [end:brute_force_search]
expect(results1.length).toBe(3);
}
// ---- Binary (hamming) vector search ----
{
const schema = new Schema([
new Field("id", new Int32(), true),
new Field("vector", new FixedSizeList(32, new Field("item", new Uint8()))),
]);
const data = lancedb.makeArrowTable(
Array(1000)
.fill(0)
.map((_, i) => ({
// the 256 bits are stored in 32 bytes; if your data is already in
// this format, you can skip the packBits step
id: i,
vector: lancedb.packBits(Array(256).fill(i % 2)),
})),
{ schema },
);
// --8<-- [start:binary_search]
const tbl = await db.createTable("binary_vectors", data, {
mode: "overwrite",
});
await tbl.createIndex("vector", {
config: lancedb.Index.ivfFlat({
numPartitions: 10,
distanceType: "hamming",
}),
});
const query = Array(32)
.fill(1)
.map(() => Math.floor(Math.random() * 255));
const results = await tbl.query().nearestTo(query).limit(10).toArray();
// --8<-- [end:binary_search]
expect(results.length).toBeLessThanOrEqual(10);
}
// ---- Enterprise-style prefilter / postfilter / batch search ----
{
const dimensions = 768;
const rows = Array.from({ length: 50 }, (_, i) => ({
vector: Array.from({ length: dimensions }, () => Math.random() * 2 - 1),
text: `story ${i}`,
keywords: `kw${i}`,
label: i % 4,
}));
await db.createTable("lancedb-enterprise-quickstart", rows, {
mode: "overwrite",
});
{
// --8<-- [start:vector_search_prefilter]
// Generate a sample 768-dimension embedding vector (typical for BERT-based models)
// In real applications, you would get this from an embedding model
const dimensions = 768;
const queryEmbed = Array.from(
{ length: dimensions },
() => Math.random() * 2 - 1,
);
// Open table and perform search
const tableName = "lancedb-enterprise-quickstart";
const table = await db.openTable(tableName);
// Vector search with filters (pre-filtering is the default)
const vectorResults = await table
.search(queryEmbed)
.where("label > 2")
.select(["text", "keywords", "label"])
.limit(5)
.toArray();
console.log("Search results (with pre-filtering):");
console.log(vectorResults);
// --8<-- [end:vector_search_prefilter]
// --8<-- [start:vector_search_postfilter]
const vectorResultsWithPostFilter = await (
table.search(queryEmbed) as lancedb.VectorQuery
)
.where("label > 2")
.postfilter()
.select(["text", "keywords", "label"])
.limit(5)
.toArray();
console.log("Vector search results with post-filter:");
console.log(vectorResultsWithPostFilter);
// --8<-- [end:vector_search_postfilter]
// --8<-- [start:batch_search]
// Batch query
console.log("Performing batch vector search...");
const batchSize = 5;
const queryVectors = Array.from({ length: batchSize }, () =>
Array.from({ length: dimensions }, () => Math.random() * 2 - 1),
);
let batchQuery = table.search(queryVectors[0]) as lancedb.VectorQuery;
for (let i = 1; i < batchSize; i++) {
batchQuery = batchQuery.addQueryVector(queryVectors[i]);
}
const batchResults = await batchQuery
.select(["text", "keywords", "label"])
.limit(5)
.toArray();
console.log("Batch vector search results:");
console.log(batchResults);
// --8<-- [end:batch_search]
expect(batchResults.length).toBeGreaterThan(0);
}
}
});
});

Some files were not shown because too many files have changed in this diff Show More