Compare commits

..

1 Commits

Author SHA1 Message Date
David Myriel
3e9f0ac784 Update mkdocs.yml 2025-08-06 17:17:45 -07:00
128 changed files with 2584 additions and 12169 deletions

View File

@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.22.1-beta.3"
current_version = "0.21.2"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.

View File

@@ -56,11 +56,22 @@ jobs:
with:
node-version: 20
cache: 'npm'
cache-dependency-path: node/package-lock.json
- name: Install node dependencies
working-directory: node
run: |
sudo apt update
sudo apt install -y protobuf-compiler libssl-dev
- name: Build node
working-directory: node
run: |
npm ci
npm run build
npm run tsc
- name: Create markdown files
working-directory: node
run: |
npx typedoc --plugin typedoc-plugin-markdown --out ../docs/src/javascript src/index.ts
- name: Build docs
working-directory: docs
run: |

View File

@@ -24,8 +24,7 @@ env:
jobs:
test-python:
name: Test doc python code
runs-on: warp-ubuntu-2204-x64-8x
timeout-minutes: 60
runs-on: ubuntu-24.04
steps:
- name: Checkout
uses: actions/checkout@v4
@@ -59,3 +58,51 @@ jobs:
run: |
cd docs/test/python
for d in *; do cd "$d"; echo "$d".py; python "$d".py; cd ..; done
test-node:
name: Test doc nodejs code
runs-on: ubuntu-24.04
timeout-minutes: 60
strategy:
fail-fast: false
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
lfs: true
- name: Print CPU capabilities
run: cat /proc/cpuinfo
- name: Set up Node
uses: actions/setup-node@v4
with:
node-version: 20
- name: Install protobuf
run: |
sudo apt update
sudo apt install -y protobuf-compiler
- name: Install dependecies needed for ubuntu
run: |
sudo apt install -y libssl-dev
rustup update && rustup default
- name: Rust cache
uses: swatinem/rust-cache@v2
- name: Install node dependencies
run: |
sudo swapoff -a
sudo fallocate -l 8G /swapfile
sudo chmod 600 /swapfile
sudo mkswap /swapfile
sudo swapon /swapfile
sudo swapon --show
cd node
npm ci
npm run build-release
cd ../docs
npm install
- name: Test
env:
LANCEDB_URI: ${{ secrets.LANCEDB_URI }}
LANCEDB_DEV_API_KEY: ${{ secrets.LANCEDB_DEV_API_KEY }}
run: |
cd docs
npm t

View File

@@ -79,7 +79,7 @@ jobs:
with:
node-version: ${{ matrix.node-version }}
cache: 'npm'
cache-dependency-path: nodejs/package-lock.json
cache-dependency-path: node/package-lock.json
- uses: Swatinem/rust-cache@v2
- name: Install dependencies
run: |
@@ -137,7 +137,7 @@ jobs:
with:
node-version: 20
cache: 'npm'
cache-dependency-path: nodejs/package-lock.json
cache-dependency-path: node/package-lock.json
- uses: Swatinem/rust-cache@v2
- name: Install dependencies
run: |

View File

@@ -56,7 +56,7 @@ jobs:
pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
fury_token: ${{ secrets.FURY_TOKEN }}
mac:
timeout-minutes: 90
timeout-minutes: 60
runs-on: ${{ matrix.config.runner }}
strategy:
matrix:
@@ -64,7 +64,7 @@ jobs:
- target: x86_64-apple-darwin
runner: macos-13
- target: aarch64-apple-darwin
runner: warp-macos-14-arm64-6x
runner: macos-14
env:
MACOSX_DEPLOYMENT_TARGET: 10.15
steps:

3
.gitignore vendored
View File

@@ -31,6 +31,9 @@ python/dist
*.node
**/node_modules
**/.DS_Store
node/dist
node/examples/**/package-lock.json
node/examples/**/dist
nodejs/lancedb/native*
dist

View File

@@ -11,70 +11,14 @@ Project layout:
* `nodejs`: The Typescript bindings, using napi-rs
* `java`: The Java bindings
(`rust/ffi` and `node/` are for a deprecated package. You can ignore them.)
Common commands:
* Check for compiler errors: `cargo check --quiet --features remote --tests --examples`
* Run tests: `cargo test --quiet --features remote --tests`
* Run specific test: `cargo test --quiet --features remote -p <package_name> --test <test_name>`
* Lint: `cargo clippy --quiet --features remote --tests --examples`
* Check for compiler errors: `cargo check --features remote --tests --examples`
* Run tests: `cargo test --features remote --tests`
* Run specific test: `cargo test --features remote -p <package_name> --test <test_name>`
* Lint: `cargo clippy --features remote --tests --examples`
* Format: `cargo fmt --all`
Before committing changes, run formatting.
## Coding tips
* When writing Rust doctests for things that require a connection or table reference,
write them as a function instead of a fully executable test. This allows type checking
to run but avoids needing a full test environment. For example:
```rust
/// ```
/// use lance_index::scalar::FullTextSearchQuery;
/// use lancedb::query::{QueryBase, ExecutableQuery};
///
/// # use lancedb::Table;
/// # async fn query(table: &Table) -> Result<(), Box<dyn std::error::Error>> {
/// let results = table.query()
/// .full_text_search(FullTextSearchQuery::new("hello world".into()))
/// .execute()
/// .await?;
/// # Ok(())
/// # }
/// ```
```
## Example plan: adding a new method on Table
Adding a new method involves first adding it to the Rust core, then exposing it
in the Python and TypeScript bindings. There are both local and remote tables.
Remote tables are implemented via a HTTP API and require the `remote` cargo
feature flag to be enabled. Python has both sync and async methods.
Rust core changes:
1. Add method on `Table` struct in `rust/lancedb/src/table.rs` (calls `BaseTable` trait).
2. Add method to `BaseTable` trait in `rust/lancedb/src/table.rs`.
3. Implement new trait method on `NativeTable` in `rust/lancedb/src/table.rs`.
* Test with unit test in `rust/lancedb/src/table.rs`.
4. Implement new trait method on `RemoteTable` in `rust/lancedb/src/remote/table.rs`.
* Test with unit test in `rust/lancedb/src/remote/table.rs` against mocked endpoint.
Python bindings changes:
1. Add PyO3 method binding in `python/src/table.rs`. Run `make develop` to compile bindings.
2. Add types for PyO3 method in `python/python/lancedb/_lancedb.pyi`.
3. Add method to `AsyncTable` class in `python/python/lancedb/table.py`.
4. Add abstract method to `Table` abstract base class in `python/python/lancedb/table.py`.
5. Add concrete sync method to `LanceTable` class in `python/python/lancedb/table.py`.
* Should use `LOOP.run()` to call the corresponding `AsyncTable` method.
6. Add concrete sync method to `RemoteTable` class in `python/python/lancedb/remote/table.py`.
7. Add unit test in `python/tests/test_table.py`.
TypeScript bindings changes:
1. Add napi-rs method binding on `Table` in `nodejs/src/table.rs`.
2. Run `npm run build` to generate TypeScript definitions.
3. Add typescript method on abstract class `Table` in `nodejs/src/table.ts`.
4. Add concrete method on `LocalTable` class in `nodejs/src/native_table.ts`.
* Note: despite the name, this class is also used for remote tables.
5. Add test in `nodejs/__test__/table.test.ts`.
6. Run `npm run docs` to generate TypeScript documentation.

1799
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,5 +1,10 @@
[workspace]
members = ["rust/lancedb", "nodejs", "python", "java/core/lancedb-jni"]
members = [
"rust/lancedb",
"nodejs",
"python",
"java/core/lancedb-jni",
]
# Python package needs to be built by maturin.
exclude = ["python"]
resolver = "2"
@@ -15,14 +20,16 @@ categories = ["database-implementations"]
rust-version = "1.78.0"
[workspace.dependencies]
lance = { "version" = "=0.37.0", default-features = false, "features" = ["dynamodb"] }
lance-io = { "version" = "=0.37.0", default-features = false }
lance-index = "=0.37.0"
lance-linalg = "=0.37.0"
lance-table = "=0.37.0"
lance-testing = "=0.37.0"
lance-datafusion = "=0.37.0"
lance-encoding = "=0.37.0"
lance = { "version" = "=0.32.1", "features" = [
"dynamodb",
], "tag" = "v0.32.1-beta.2", "git" = "https://github.com/lancedb/lance.git" }
lance-io = { "version" = "=0.32.1", "tag" = "v0.32.1-beta.2", "git" = "https://github.com/lancedb/lance.git" }
lance-index = { "version" = "=0.32.1", "tag" = "v0.32.1-beta.2", "git" = "https://github.com/lancedb/lance.git" }
lance-linalg = { "version" = "=0.32.1", "tag" = "v0.32.1-beta.2", "git" = "https://github.com/lancedb/lance.git" }
lance-table = { "version" = "=0.32.1", "tag" = "v0.32.1-beta.2", "git" = "https://github.com/lancedb/lance.git" }
lance-testing = { "version" = "=0.32.1", "tag" = "v0.32.1-beta.2", "git" = "https://github.com/lancedb/lance.git" }
lance-datafusion = { "version" = "=0.32.1", "tag" = "v0.32.1-beta.2", "git" = "https://github.com/lancedb/lance.git" }
lance-encoding = { "version" = "=0.32.1", "tag" = "v0.32.1-beta.2", "git" = "https://github.com/lancedb/lance.git" }
# Note that this one does not include pyarrow
arrow = { version = "55.1", optional = false }
arrow-array = "55.1"
@@ -33,12 +40,12 @@ arrow-schema = "55.1"
arrow-arith = "55.1"
arrow-cast = "55.1"
async-trait = "0"
datafusion = { version = "49.0", default-features = false }
datafusion-catalog = "49.0"
datafusion-common = { version = "49.0", default-features = false }
datafusion-execution = "49.0"
datafusion-expr = "49.0"
datafusion-physical-plan = "49.0"
datafusion = { version = "48.0", default-features = false }
datafusion-catalog = "48.0"
datafusion-common = { version = "48.0", default-features = false }
datafusion-execution = "48.0"
datafusion-expr = "48.0"
datafusion-physical-plan = "48.0"
env_logger = "0.11"
half = { "version" = "2.6.0", default-features = false, features = [
"num-traits",
@@ -55,11 +62,12 @@ rand = "0.9"
regex = "1.10"
lazy_static = "1"
semver = "1.0.25"
crunchy = "0.2.4"
# Temporary pins to work around downstream issues
# https://github.com/apache/arrow-rs/commit/2fddf85afcd20110ce783ed5b4cdeb82293da30b
chrono = "=0.4.41"
# https://github.com/RustCrypto/formats/issues/1684
base64ct = "=1.6.0"
# Workaround for: https://github.com/eira-fransham/crunchy/issues/13
crunchy = "=0.2.2"
# Workaround for: https://github.com/Lokathor/bytemuck/issues/306
bytemuck_derive = ">=1.8.1, <1.9.0"

View File

@@ -1,5 +1,4 @@
import argparse
import re
import sys
import json
@@ -19,12 +18,8 @@ def run_command(command: str) -> str:
def get_latest_stable_version() -> str:
version_line = run_command("cargo info lance | grep '^version:'")
# Example output: "version: 0.35.0 (latest 0.37.0)"
match = re.search(r'\(latest ([0-9.]+)\)', version_line)
if match:
return match.group(1)
# Fallback: use the first version after 'version:'
return version_line.split("version:")[1].split()[0].strip()
version = version_line.split(" ")[1].strip()
return version
def get_latest_preview_version() -> str:
@@ -59,52 +54,6 @@ def extract_features(line: str) -> list:
return []
def extract_default_features(line: str) -> bool:
"""
Checks if default-features = false is present in a line in Cargo.toml.
Example: 'lance = { "version" = "=0.29.0", default-features = false, "features" = ["dynamodb"] }'
Returns: True if default-features = false is present, False otherwise
"""
import re
match = re.search(r'default-features\s*=\s*false', line)
return match is not None
def dict_to_toml_line(package_name: str, config: dict) -> str:
"""
Converts a configuration dictionary to a TOML dependency line.
Dictionary insertion order is preserved (Python 3.7+), so the caller
controls the order of fields in the output.
Args:
package_name: The name of the package (e.g., "lance", "lance-io")
config: Dictionary with keys like "version", "path", "git", "tag", "features", "default-features"
The order of keys in this dict determines the order in the output.
Returns:
A properly formatted TOML line with a trailing newline
"""
# If only version is specified, use simple format
if len(config) == 1 and "version" in config:
return f'{package_name} = "{config["version"]}"\n'
# Otherwise, use inline table format
parts = []
for key, value in config.items():
if key == "default-features" and not value:
parts.append("default-features = false")
elif key == "features":
parts.append(f'"features" = {json.dumps(value)}')
elif isinstance(value, str):
parts.append(f'"{key}" = "{value}"')
else:
# This shouldn't happen with our current usage
parts.append(f'"{key}" = {json.dumps(value)}')
return f'{package_name} = {{ {", ".join(parts)} }}\n'
def update_cargo_toml(line_updater):
"""
Updates the Cargo.toml file by applying the line_updater function to each line.
@@ -118,27 +67,20 @@ def update_cargo_toml(line_updater):
is_parsing_lance_line = False
for line in lines:
if line.startswith("lance"):
# Check if this is a single-line or multi-line entry
# Single-line entries either:
# 1. End with } (complete inline table)
# 2. End with " (simple version string)
# Multi-line entries start with { but don't end with }
if line.strip().endswith("}") or line.strip().endswith('"'):
# Single-line entry - process immediately
# Update the line using the provided function
if line.strip().endswith("}"):
new_lines.append(line_updater(line))
elif "{" in line and not line.strip().endswith("}"):
# Multi-line entry - start accumulating
else:
lance_line = line
is_parsing_lance_line = True
else:
# Single-line entry without quotes or braces (shouldn't happen but handle it)
new_lines.append(line_updater(line))
elif is_parsing_lance_line:
lance_line += line
if line.strip().endswith("}"):
new_lines.append(line_updater(lance_line))
lance_line = ""
is_parsing_lance_line = False
else:
print("doesn't end with }:", line)
else:
# Keep the line unchanged
new_lines.append(line)
@@ -150,25 +92,18 @@ def update_cargo_toml(line_updater):
def set_stable_version(version: str):
"""
Sets lines to
lance = { "version" = "=0.29.0", default-features = false, "features" = ["dynamodb"] }
lance-io = { "version" = "=0.29.0", default-features = false }
lance = { "version" = "=0.29.0", "features" = ["dynamodb"] }
lance-io = "=0.29.0"
...
"""
def line_updater(line: str) -> str:
package_name = line.split("=", maxsplit=1)[0].strip()
# Build config in desired order: version, default-features, features
config = {"version": f"={version}"}
if extract_default_features(line):
config["default-features"] = False
features = extract_features(line)
if features:
config["features"] = features
return dict_to_toml_line(package_name, config)
return f'{package_name} = {{ "version" = "={version}", "features" = {json.dumps(features)} }}\n'
else:
return f'{package_name} = "={version}"\n'
update_cargo_toml(line_updater)
@@ -176,29 +111,19 @@ def set_stable_version(version: str):
def set_preview_version(version: str):
"""
Sets lines to
lance = { "version" = "=0.29.0", default-features = false, "features" = ["dynamodb"], "tag" = "v0.29.0-beta.2", "git" = "https://github.com/lancedb/lance.git" }
lance-io = { "version" = "=0.29.0", default-features = false, "tag" = "v0.29.0-beta.2", "git" = "https://github.com/lancedb/lance.git" }
lance = { "version" = "=0.29.0", "features" = ["dynamodb"], tag = "v0.29.0-beta.2", git="https://github.com/lancedb/lance.git" }
lance-io = { version = "=0.29.0", tag = "v0.29.0-beta.2", git="https://github.com/lancedb/lance.git" }
...
"""
def line_updater(line: str) -> str:
package_name = line.split("=", maxsplit=1)[0].strip()
base_version = version.split("-")[0] # Get the base version without beta suffix
# Build config in desired order: version, default-features, features, tag, git
config = {"version": f"={base_version}"}
if extract_default_features(line):
config["default-features"] = False
features = extract_features(line)
base_version = version.split("-")[0] # Get the base version without beta suffix
if features:
config["features"] = features
config["tag"] = f"v{version}"
config["git"] = "https://github.com/lancedb/lance.git"
return dict_to_toml_line(package_name, config)
return f'{package_name} = {{ "version" = "={base_version}", "features" = {json.dumps(features)}, "tag" = "v{version}", "git" = "https://github.com/lancedb/lance.git" }}\n'
else:
return f'{package_name} = {{ "version" = "={base_version}", "tag" = "v{version}", "git" = "https://github.com/lancedb/lance.git" }}\n'
update_cargo_toml(line_updater)
@@ -206,25 +131,18 @@ def set_preview_version(version: str):
def set_local_version():
"""
Sets lines to
lance = { "path" = "../lance/rust/lance", default-features = false, "features" = ["dynamodb"] }
lance-io = { "path" = "../lance/rust/lance-io", default-features = false }
lance = { path = "../lance/rust/lance", features = ["dynamodb"] }
lance-io = { path = "../lance/rust/lance-io" }
...
"""
def line_updater(line: str) -> str:
package_name = line.split("=", maxsplit=1)[0].strip()
# Build config in desired order: path, default-features, features
config = {"path": f"../lance/rust/{package_name}"}
if extract_default_features(line):
config["default-features"] = False
features = extract_features(line)
if features:
config["features"] = features
return dict_to_toml_line(package_name, config)
return f'{package_name} = {{ "path" = "../lance/rust/{package_name}", "features" = {json.dumps(features)} }}\n'
else:
return f'{package_name} = {{ "path" = "../lance/rust/{package_name}" }}\n'
update_cargo_toml(line_updater)

View File

@@ -15,13 +15,16 @@ cargo metadata --quiet > /dev/null
pushd nodejs || exit 1
npm install --package-lock-only --silent
popd
pushd node || exit 1
npm install --package-lock-only --silent
popd
if git diff --quiet --exit-code; then
echo "No lockfile changes to commit; skipping amend."
elif $AMEND; then
git add Cargo.lock nodejs/package-lock.json
git add Cargo.lock nodejs/package-lock.json node/package-lock.json
git commit --amend --no-edit
else
git add Cargo.lock nodejs/package-lock.json
git add Cargo.lock nodejs/package-lock.json node/package-lock.json
git commit -m "Update lockfiles"
fi

View File

@@ -103,264 +103,6 @@ markdown_extensions:
permalink: ""
nav:
- Home:
- LanceDB: index.md
- 🏃🏼‍♂️ Quick start: basic.md
- 📚 Concepts:
- Vector search: concepts/vector_search.md
- Indexing:
- IVFPQ: concepts/index_ivfpq.md
- HNSW: concepts/index_hnsw.md
- Storage: concepts/storage.md
- Data management: concepts/data_management.md
- 🔨 Guides:
- Working with tables: guides/tables.md
- Building a vector index: ann_indexes.md
- Vector Search: search.md
- Full-text search (native): fts.md
- Full-text search (tantivy-based): fts_tantivy.md
- Building a scalar index: guides/scalar_index.md
- Hybrid search:
- Overview: hybrid_search/hybrid_search.md
- Comparing Rerankers: hybrid_search/eval.md
- Airbnb financial data example: notebooks/hybrid_search.ipynb
- Late interaction with MultiVector search:
- Overview: guides/multi-vector.md
- Example: notebooks/Multivector_on_LanceDB.ipynb
- RAG:
- Vanilla RAG: rag/vanilla_rag.md
- Multi-head RAG: rag/multi_head_rag.md
- Corrective RAG: rag/corrective_rag.md
- Agentic RAG: rag/agentic_rag.md
- Graph RAG: rag/graph_rag.md
- Self RAG: rag/self_rag.md
- Adaptive RAG: rag/adaptive_rag.md
- SFR RAG: rag/sfr_rag.md
- Advanced Techniques:
- HyDE: rag/advanced_techniques/hyde.md
- FLARE: rag/advanced_techniques/flare.md
- Reranking:
- Quickstart: reranking/index.md
- Cohere Reranker: reranking/cohere.md
- Linear Combination Reranker: reranking/linear_combination.md
- Reciprocal Rank Fusion Reranker: reranking/rrf.md
- Cross Encoder Reranker: reranking/cross_encoder.md
- ColBERT Reranker: reranking/colbert.md
- Jina Reranker: reranking/jina.md
- OpenAI Reranker: reranking/openai.md
- AnswerDotAi Rerankers: reranking/answerdotai.md
- Voyage AI Rerankers: reranking/voyageai.md
- Building Custom Rerankers: reranking/custom_reranker.md
- Example: notebooks/lancedb_reranking.ipynb
- Filtering: sql.md
- Versioning & Reproducibility:
- sync API: notebooks/reproducibility.ipynb
- async API: notebooks/reproducibility_async.ipynb
- Configuring Storage: guides/storage.md
- Migration Guide: migration.md
- Tuning retrieval performance:
- Choosing right query type: guides/tuning_retrievers/1_query_types.md
- Reranking: guides/tuning_retrievers/2_reranking.md
- Embedding fine-tuning: guides/tuning_retrievers/3_embed_tuning.md
- 🧬 Managing embeddings:
- Understand Embeddings: embeddings/understanding_embeddings.md
- Get Started: embeddings/index.md
- Embedding functions: embeddings/embedding_functions.md
- Available models:
- Overview: embeddings/default_embedding_functions.md
- Text Embedding Functions:
- Sentence Transformers: embeddings/available_embedding_models/text_embedding_functions/sentence_transformers.md
- Huggingface Embedding Models: embeddings/available_embedding_models/text_embedding_functions/huggingface_embedding.md
- Ollama Embeddings: embeddings/available_embedding_models/text_embedding_functions/ollama_embedding.md
- OpenAI Embeddings: embeddings/available_embedding_models/text_embedding_functions/openai_embedding.md
- Instructor Embeddings: embeddings/available_embedding_models/text_embedding_functions/instructor_embedding.md
- Gemini Embeddings: embeddings/available_embedding_models/text_embedding_functions/gemini_embedding.md
- Cohere Embeddings: embeddings/available_embedding_models/text_embedding_functions/cohere_embedding.md
- Jina Embeddings: embeddings/available_embedding_models/text_embedding_functions/jina_embedding.md
- AWS Bedrock Text Embedding Functions: embeddings/available_embedding_models/text_embedding_functions/aws_bedrock_embedding.md
- IBM watsonx.ai Embeddings: embeddings/available_embedding_models/text_embedding_functions/ibm_watsonx_ai_embedding.md
- Voyage AI Embeddings: embeddings/available_embedding_models/text_embedding_functions/voyageai_embedding.md
- Multimodal Embedding Functions:
- OpenClip embeddings: embeddings/available_embedding_models/multimodal_embedding_functions/openclip_embedding.md
- Imagebind embeddings: embeddings/available_embedding_models/multimodal_embedding_functions/imagebind_embedding.md
- Jina Embeddings: embeddings/available_embedding_models/multimodal_embedding_functions/jina_multimodal_embedding.md
- User-defined embedding functions: embeddings/custom_embedding_function.md
- Variables and secrets: embeddings/variables_and_secrets.md
- "Example: Multi-lingual semantic search": notebooks/multi_lingual_example.ipynb
- "Example: MultiModal CLIP Embeddings": notebooks/DisappearingEmbeddingFunction.ipynb
- 🔌 Integrations:
- Tools and data formats: integrations/index.md
- Pandas and PyArrow: python/pandas_and_pyarrow.md
- Polars: python/polars_arrow.md
- DuckDB: python/duckdb.md
- Datafusion: python/datafusion.md
- LangChain:
- LangChain 🔗: integrations/langchain.md
- LangChain demo: notebooks/langchain_demo.ipynb
- LangChain JS/TS 🔗: https://js.langchain.com/docs/integrations/vectorstores/lancedb
- LlamaIndex 🦙:
- LlamaIndex docs: integrations/llamaIndex.md
- LlamaIndex demo: notebooks/llamaIndex_demo.ipynb
- Pydantic: python/pydantic.md
- Voxel51: integrations/voxel51.md
- PromptTools: integrations/prompttools.md
- dlt: integrations/dlt.md
- phidata: integrations/phidata.md
- Genkit: integrations/genkit.md
- 🎯 Examples:
- Overview: examples/index.md
- 🐍 Python:
- Overview: examples/examples_python.md
- Build From Scratch: examples/python_examples/build_from_scratch.md
- Multimodal: examples/python_examples/multimodal.md
- Rag: examples/python_examples/rag.md
- Vector Search: examples/python_examples/vector_search.md
- Chatbot: examples/python_examples/chatbot.md
- Evaluation: examples/python_examples/evaluations.md
- AI Agent: examples/python_examples/aiagent.md
- Recommender System: examples/python_examples/recommendersystem.md
- Miscellaneous:
- Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
- Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
- 👾 JavaScript:
- Overview: examples/examples_js.md
- Serverless Website Chatbot: examples/serverless_website_chatbot.md
- YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
- TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
- 🦀 Rust:
- Overview: examples/examples_rust.md
- 📓 Studies:
- ↗Improve retrievers with hybrid search and reranking: https://blog.lancedb.com/hybrid-search-and-reranking-report/
- 💭 FAQs: faq.md
- 🔍 Troubleshooting: troubleshooting.md
- ⚙️ API reference:
- 🐍 Python: python/python.md
- 👾 JavaScript (vectordb): javascript/modules.md
- 👾 JavaScript (lancedb): js/globals.md
- 🦀 Rust: https://docs.rs/lancedb/latest/lancedb/
- Quick start: basic.md
- Concepts:
- Vector search: concepts/vector_search.md
- Indexing:
- IVFPQ: concepts/index_ivfpq.md
- HNSW: concepts/index_hnsw.md
- Storage: concepts/storage.md
- Data management: concepts/data_management.md
- Guides:
- Working with tables: guides/tables.md
- Working with SQL: guides/sql_querying.md
- Building an ANN index: ann_indexes.md
- Vector Search: search.md
- Full-text search (native): fts.md
- Full-text search (tantivy-based): fts_tantivy.md
- Building a scalar index: guides/scalar_index.md
- Hybrid search:
- Overview: hybrid_search/hybrid_search.md
- Comparing Rerankers: hybrid_search/eval.md
- Airbnb financial data example: notebooks/hybrid_search.ipynb
- Late interaction with MultiVector search:
- Overview: guides/multi-vector.md
- Document search Example: notebooks/Multivector_on_LanceDB.ipynb
- RAG:
- Vanilla RAG: rag/vanilla_rag.md
- Multi-head RAG: rag/multi_head_rag.md
- Corrective RAG: rag/corrective_rag.md
- Agentic RAG: rag/agentic_rag.md
- Graph RAG: rag/graph_rag.md
- Self RAG: rag/self_rag.md
- Adaptive RAG: rag/adaptive_rag.md
- SFR RAG: rag/sfr_rag.md
- Advanced Techniques:
- HyDE: rag/advanced_techniques/hyde.md
- FLARE: rag/advanced_techniques/flare.md
- Reranking:
- Quickstart: reranking/index.md
- Cohere Reranker: reranking/cohere.md
- Linear Combination Reranker: reranking/linear_combination.md
- Reciprocal Rank Fusion Reranker: reranking/rrf.md
- Cross Encoder Reranker: reranking/cross_encoder.md
- ColBERT Reranker: reranking/colbert.md
- Jina Reranker: reranking/jina.md
- OpenAI Reranker: reranking/openai.md
- AnswerDotAi Rerankers: reranking/answerdotai.md
- Building Custom Rerankers: reranking/custom_reranker.md
- Example: notebooks/lancedb_reranking.ipynb
- Filtering: sql.md
- Versioning & Reproducibility:
- sync API: notebooks/reproducibility.ipynb
- async API: notebooks/reproducibility_async.ipynb
- Configuring Storage: guides/storage.md
- Migration Guide: migration.md
- Tuning retrieval performance:
- Choosing right query type: guides/tuning_retrievers/1_query_types.md
- Reranking: guides/tuning_retrievers/2_reranking.md
- Embedding fine-tuning: guides/tuning_retrievers/3_embed_tuning.md
- Managing Embeddings:
- Understand Embeddings: embeddings/understanding_embeddings.md
- Get Started: embeddings/index.md
- Embedding functions: embeddings/embedding_functions.md
- Available models:
- Overview: embeddings/default_embedding_functions.md
- Text Embedding Functions:
- Sentence Transformers: embeddings/available_embedding_models/text_embedding_functions/sentence_transformers.md
- Huggingface Embedding Models: embeddings/available_embedding_models/text_embedding_functions/huggingface_embedding.md
- Ollama Embeddings: embeddings/available_embedding_models/text_embedding_functions/ollama_embedding.md
- OpenAI Embeddings: embeddings/available_embedding_models/text_embedding_functions/openai_embedding.md
- Instructor Embeddings: embeddings/available_embedding_models/text_embedding_functions/instructor_embedding.md
- Gemini Embeddings: embeddings/available_embedding_models/text_embedding_functions/gemini_embedding.md
- Cohere Embeddings: embeddings/available_embedding_models/text_embedding_functions/cohere_embedding.md
- Jina Embeddings: embeddings/available_embedding_models/text_embedding_functions/jina_embedding.md
- AWS Bedrock Text Embedding Functions: embeddings/available_embedding_models/text_embedding_functions/aws_bedrock_embedding.md
- IBM watsonx.ai Embeddings: embeddings/available_embedding_models/text_embedding_functions/ibm_watsonx_ai_embedding.md
- Multimodal Embedding Functions:
- OpenClip embeddings: embeddings/available_embedding_models/multimodal_embedding_functions/openclip_embedding.md
- Imagebind embeddings: embeddings/available_embedding_models/multimodal_embedding_functions/imagebind_embedding.md
- Jina Embeddings: embeddings/available_embedding_models/multimodal_embedding_functions/jina_multimodal_embedding.md
- User-defined embedding functions: embeddings/custom_embedding_function.md
- Variables and secrets: embeddings/variables_and_secrets.md
- "Example: Multi-lingual semantic search": notebooks/multi_lingual_example.ipynb
- "Example: MultiModal CLIP Embeddings": notebooks/DisappearingEmbeddingFunction.ipynb
- Integrations:
- Overview: integrations/index.md
- Pandas and PyArrow: python/pandas_and_pyarrow.md
- Polars: python/polars_arrow.md
- DuckDB: python/duckdb.md
- Datafusion: python/datafusion.md
- LangChain 🦜️🔗↗: integrations/langchain.md
- LangChain.js 🦜️🔗↗: https://js.langchain.com/docs/integrations/vectorstores/lancedb
- LlamaIndex 🦙↗: integrations/llamaIndex.md
- Pydantic: python/pydantic.md
- Voxel51: integrations/voxel51.md
- PromptTools: integrations/prompttools.md
- dlt: integrations/dlt.md
- phidata: integrations/phidata.md
- Genkit: integrations/genkit.md
- Examples:
- examples/index.md
- 🐍 Python:
- Overview: examples/examples_python.md
- Build From Scratch: examples/python_examples/build_from_scratch.md
- Multimodal: examples/python_examples/multimodal.md
- Rag: examples/python_examples/rag.md
- Vector Search: examples/python_examples/vector_search.md
- Chatbot: examples/python_examples/chatbot.md
- Evaluation: examples/python_examples/evaluations.md
- AI Agent: examples/python_examples/aiagent.md
- Recommender System: examples/python_examples/recommendersystem.md
- Miscellaneous:
- Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
- Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
- 👾 JavaScript:
- Overview: examples/examples_js.md
- Serverless Website Chatbot: examples/serverless_website_chatbot.md
- YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
- TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
- 🦀 Rust:
- Overview: examples/examples_rust.md
- Studies:
- studies/overview.md
- ↗Improve retrievers with hybrid search and reranking: https://blog.lancedb.com/hybrid-search-and-reranking-report/
- API reference:
- Overview: api_reference.md
- Python: python/python.md

View File

@@ -13,7 +13,7 @@ The following concepts are important to keep in mind:
- Data is versioned, with each insert operation creating a new version of the dataset and an update to the manifest that tracks versions via metadata
!!! note
1. First, each version contains metadata and just the new/updated data in your transaction. So if you have 100 versions, they aren't 100 duplicates of the same data. However, they do have 100x the metadata overhead of a single version, which can result in slower queries.
1. First, each version contains metadata and just the new/updated data in your transaction. So if you have 100 versions, they aren't 100 duplicates of the same data. However, they do have 100x the metadata overhead of a single version, which can result in slower queries.
2. Second, these versions exist to keep LanceDB scalable and consistent. We do not immediately blow away old versions when creating new ones because other clients might be in the middle of querying the old version. It's important to retain older versions for as long as they might be queried.
## What are fragments?
@@ -37,10 +37,6 @@ Depending on the use case and dataset, optimal compaction will have different re
- Its always better to use *batch* inserts rather than adding 1 row at a time (to avoid too small fragments). If single-row inserts are unavoidable, run compaction on a regular basis to merge them into larger fragments.
- Keep the number of fragments under 100, which is suitable for most use cases (for *really* large datasets of >500M rows, more fragments might be needed)
!!! note
LanceDB Cloud/Enterprise supports [auto-compaction](https://docs.lancedb.com/enterprise/architecture/architecture#write-path) which automatically optimizes fragments in the background as data changes.
## Deletion
Although Lance allows you to delete rows from a dataset, it does not actually delete the data immediately. It simply marks the row as deleted in the `DataFile` that represents a fragment. For a given version of the dataset, each fragment can have up to one deletion file (if no rows were ever deleted from that fragment, it will not have a deletion file). This is important to keep in mind because it means that the data is still there, and can be recovered if needed, as long as that version still exists based on your backup policy.
@@ -54,9 +50,13 @@ Reindexing is the process of updating the index to account for new data, keeping
Both LanceDB OSS and Cloud support reindexing, but the process (at least for now) is different for each, depending on the type of index.
In LanceDB OSS, re-indexing happens synchronously when you call either `create_index` or `optimize` on a table. In LanceDB Cloud, re-indexing happens asynchronously as you add and update data in your table.
When a reindex job is triggered in the background, the entire data is reindexed, but in the interim as new queries come in, LanceDB will combine results from the existing index with exhaustive kNN search on the new data. This is done to ensure that you're still searching on all your data, but it does come at a performance cost. The more data that you add without reindexing, the impact on latency (due to exhaustive search) can be noticeable.
By default, queries will search new data even if it has yet to be indexed. This is done using brute-force methods, such as kNN for vector search, and combined with the fast index search results. This is done to ensure that you're always searching over all your data, but it does come at a performance cost. Without reindexing, adding more data to a table will make queries slower and more expensive. This behavior can be disabled by setting the [fast_search](https://lancedb.github.io/lancedb/python/python/#lancedb.query.AsyncQuery.fast_search) parameter which will instruct the query to ignore un-indexed data.
### Vector reindex
* LanceDB Cloud/Enterprise supports [automatic incremental reindexing](https://docs.lancedb.com/core#vector-index) for vector, scalar, and FTS indices, where a background process will trigger a new index build for you automatically when new data is added or modified in a dataset
* LanceDB Cloud supports incremental reindexing, where a background process will trigger a new index build for you automatically when new data is added to a dataset
* LanceDB OSS requires you to manually trigger a reindex operation -- we are working on adding incremental reindexing to LanceDB OSS as well
### FTS reindex
FTS reindexing is supported in both LanceDB OSS and Cloud, but requires that it's manually rebuilt once you have a significant enough amount of new data added that needs to be reindexed. We [updated](https://github.com/lancedb/lancedb/pull/762) Tantivy's default heap size from 128MB to 1GB in LanceDB to make it much faster to reindex, by up to 10x from the default settings.

View File

@@ -45,8 +45,6 @@ Any attempt to use the connection after it is closed will result in an error.
### createEmptyTable()
#### createEmptyTable(name, schema, options)
```ts
abstract createEmptyTable(
name,
@@ -56,7 +54,7 @@ abstract createEmptyTable(
Creates a new empty Table
##### Parameters
#### Parameters
* **name**: `string`
The name of the table.
@@ -65,39 +63,8 @@ Creates a new empty Table
The schema of the table
* **options?**: `Partial`&lt;[`CreateTableOptions`](../interfaces/CreateTableOptions.md)&gt;
Additional options (backwards compatibility)
##### Returns
`Promise`&lt;[`Table`](Table.md)&gt;
#### createEmptyTable(name, schema, namespace, options)
```ts
abstract createEmptyTable(
name,
schema,
namespace?,
options?): Promise<Table>
```
Creates a new empty Table
##### Parameters
* **name**: `string`
The name of the table.
* **schema**: [`SchemaLike`](../type-aliases/SchemaLike.md)
The schema of the table
* **namespace?**: `string`[]
The namespace to create the table in (defaults to root namespace)
* **options?**: `Partial`&lt;[`CreateTableOptions`](../interfaces/CreateTableOptions.md)&gt;
Additional options
##### Returns
#### Returns
`Promise`&lt;[`Table`](Table.md)&gt;
@@ -105,10 +72,10 @@ Creates a new empty Table
### createTable()
#### createTable(options, namespace)
#### createTable(options)
```ts
abstract createTable(options, namespace?): Promise<Table>
abstract createTable(options): Promise<Table>
```
Creates a new Table and initialize it with new data.
@@ -118,9 +85,6 @@ Creates a new Table and initialize it with new data.
* **options**: `object` & `Partial`&lt;[`CreateTableOptions`](../interfaces/CreateTableOptions.md)&gt;
The options object.
* **namespace?**: `string`[]
The namespace to create the table in (defaults to root namespace)
##### Returns
`Promise`&lt;[`Table`](Table.md)&gt;
@@ -146,38 +110,6 @@ Creates a new Table and initialize it with new data.
to be inserted into the table
* **options?**: `Partial`&lt;[`CreateTableOptions`](../interfaces/CreateTableOptions.md)&gt;
Additional options (backwards compatibility)
##### Returns
`Promise`&lt;[`Table`](Table.md)&gt;
#### createTable(name, data, namespace, options)
```ts
abstract createTable(
name,
data,
namespace?,
options?): Promise<Table>
```
Creates a new Table and initialize it with new data.
##### Parameters
* **name**: `string`
The name of the table.
* **data**: [`TableLike`](../type-aliases/TableLike.md) \| `Record`&lt;`string`, `unknown`&gt;[]
Non-empty Array of Records
to be inserted into the table
* **namespace?**: `string`[]
The namespace to create the table in (defaults to root namespace)
* **options?**: `Partial`&lt;[`CreateTableOptions`](../interfaces/CreateTableOptions.md)&gt;
Additional options
##### Returns
@@ -202,16 +134,11 @@ Return a brief description of the connection
### dropAllTables()
```ts
abstract dropAllTables(namespace?): Promise<void>
abstract dropAllTables(): Promise<void>
```
Drop all tables in the database.
#### Parameters
* **namespace?**: `string`[]
The namespace to drop tables from (defaults to root namespace).
#### Returns
`Promise`&lt;`void`&gt;
@@ -221,7 +148,7 @@ Drop all tables in the database.
### dropTable()
```ts
abstract dropTable(name, namespace?): Promise<void>
abstract dropTable(name): Promise<void>
```
Drop an existing table.
@@ -231,9 +158,6 @@ Drop an existing table.
* **name**: `string`
The name of the table to drop.
* **namespace?**: `string`[]
The namespace of the table (defaults to root namespace).
#### Returns
`Promise`&lt;`void`&gt;
@@ -257,10 +181,7 @@ Return true if the connection has not been closed
### openTable()
```ts
abstract openTable(
name,
namespace?,
options?): Promise<Table>
abstract openTable(name, options?): Promise<Table>
```
Open a table in the database.
@@ -270,11 +191,7 @@ Open a table in the database.
* **name**: `string`
The name of the table
* **namespace?**: `string`[]
The namespace of the table (defaults to root namespace)
* **options?**: `Partial`&lt;[`OpenTableOptions`](../interfaces/OpenTableOptions.md)&gt;
Additional options
#### Returns
@@ -284,8 +201,6 @@ Open a table in the database.
### tableNames()
#### tableNames(options)
```ts
abstract tableNames(options?): Promise<string[]>
```
@@ -294,35 +209,12 @@ List all the table names in this database.
Tables will be returned in lexicographical order.
##### Parameters
* **options?**: `Partial`&lt;[`TableNamesOptions`](../interfaces/TableNamesOptions.md)&gt;
options to control the
paging / start point (backwards compatibility)
##### Returns
`Promise`&lt;`string`[]&gt;
#### tableNames(namespace, options)
```ts
abstract tableNames(namespace?, options?): Promise<string[]>
```
List all the table names in this database.
Tables will be returned in lexicographical order.
##### Parameters
* **namespace?**: `string`[]
The namespace to list tables from (defaults to root namespace)
#### Parameters
* **options?**: `Partial`&lt;[`TableNamesOptions`](../interfaces/TableNamesOptions.md)&gt;
options to control the
paging / start point
##### Returns
#### Returns
`Promise`&lt;`string`[]&gt;

View File

@@ -1,85 +0,0 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / HeaderProvider
# Class: `abstract` HeaderProvider
Abstract base class for providing custom headers for each request.
Users can implement this interface to provide dynamic headers for various purposes
such as authentication (OAuth tokens, API keys), request tracking (correlation IDs),
custom metadata, or any other header-based requirements. The provider is called
before each request to ensure fresh header values are always used.
## Examples
Simple JWT token provider:
```typescript
class JWTProvider extends HeaderProvider {
constructor(private token: string) {
super();
}
getHeaders(): Record<string, string> {
return { authorization: `Bearer ${this.token}` };
}
}
```
Provider with request tracking:
```typescript
class RequestTrackingProvider extends HeaderProvider {
constructor(private sessionId: string) {
super();
}
getHeaders(): Record<string, string> {
return {
"X-Session-Id": this.sessionId,
"X-Request-Id": `req-${Date.now()}`
};
}
}
```
## Extended by
- [`StaticHeaderProvider`](StaticHeaderProvider.md)
- [`OAuthHeaderProvider`](OAuthHeaderProvider.md)
## Constructors
### new HeaderProvider()
```ts
new HeaderProvider(): HeaderProvider
```
#### Returns
[`HeaderProvider`](HeaderProvider.md)
## Methods
### getHeaders()
```ts
abstract getHeaders(): Record<string, string>
```
Get the latest headers to be added to requests.
This method is called before each request to the remote LanceDB server.
Implementations should return headers that will be merged with existing headers.
#### Returns
`Record`&lt;`string`, `string`&gt;
Dictionary of header names to values to add to the request.
#### Throws
If unable to fetch headers, the exception will be propagated and the request will fail.

View File

@@ -1,29 +0,0 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / NativeJsHeaderProvider
# Class: NativeJsHeaderProvider
JavaScript HeaderProvider implementation that wraps a JavaScript callback.
This is the only native header provider - all header provider implementations
should provide a JavaScript function that returns headers.
## Constructors
### new NativeJsHeaderProvider()
```ts
new NativeJsHeaderProvider(getHeadersCallback): NativeJsHeaderProvider
```
Create a new JsHeaderProvider from a JavaScript callback
#### Parameters
* **getHeadersCallback**
#### Returns
[`NativeJsHeaderProvider`](NativeJsHeaderProvider.md)

View File

@@ -1,108 +0,0 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / OAuthHeaderProvider
# Class: OAuthHeaderProvider
Example implementation: OAuth token provider with automatic refresh.
This is an example implementation showing how to manage OAuth tokens
with automatic refresh when they expire.
## Example
```typescript
async function fetchToken(): Promise<TokenResponse> {
const response = await fetch("https://oauth.example.com/token", {
method: "POST",
body: JSON.stringify({
grant_type: "client_credentials",
client_id: "your-client-id",
client_secret: "your-client-secret"
}),
headers: { "Content-Type": "application/json" }
});
const data = await response.json();
return {
accessToken: data.access_token,
expiresIn: data.expires_in
};
}
const provider = new OAuthHeaderProvider(fetchToken);
const headers = provider.getHeaders();
// Returns: {"authorization": "Bearer <your-token>"}
```
## Extends
- [`HeaderProvider`](HeaderProvider.md)
## Constructors
### new OAuthHeaderProvider()
```ts
new OAuthHeaderProvider(tokenFetcher, refreshBufferSeconds): OAuthHeaderProvider
```
Initialize the OAuth provider.
#### Parameters
* **tokenFetcher**
Function to fetch new tokens. Should return object with 'accessToken' and optionally 'expiresIn'.
* **refreshBufferSeconds**: `number` = `300`
Seconds before expiry to refresh token. Default 300 (5 minutes).
#### Returns
[`OAuthHeaderProvider`](OAuthHeaderProvider.md)
#### Overrides
[`HeaderProvider`](HeaderProvider.md).[`constructor`](HeaderProvider.md#constructors)
## Methods
### getHeaders()
```ts
getHeaders(): Record<string, string>
```
Get OAuth headers, refreshing token if needed.
Note: This is synchronous for now as the Rust implementation expects sync.
In a real implementation, this would need to handle async properly.
#### Returns
`Record`&lt;`string`, `string`&gt;
Headers with Bearer token authorization.
#### Throws
If unable to fetch or refresh token.
#### Overrides
[`HeaderProvider`](HeaderProvider.md).[`getHeaders`](HeaderProvider.md#getheaders)
***
### refreshToken()
```ts
refreshToken(): Promise<void>
```
Manually refresh the token.
Call this before using getHeaders() to ensure token is available.
#### Returns
`Promise`&lt;`void`&gt;

View File

@@ -14,7 +14,7 @@ A builder for LanceDB queries.
## Extends
- `StandardQueryBase`&lt;`NativeQuery`&gt;
- [`QueryBase`](QueryBase.md)&lt;`NativeQuery`&gt;
## Properties
@@ -26,7 +26,7 @@ protected inner: Query | Promise<Query>;
#### Inherited from
`StandardQueryBase.inner`
[`QueryBase`](QueryBase.md).[`inner`](QueryBase.md#inner)
## Methods
@@ -73,7 +73,7 @@ AnalyzeExec verbose=true, metrics=[]
#### Inherited from
`StandardQueryBase.analyzePlan`
[`QueryBase`](QueryBase.md).[`analyzePlan`](QueryBase.md#analyzeplan)
***
@@ -107,7 +107,7 @@ single query)
#### Inherited from
`StandardQueryBase.execute`
[`QueryBase`](QueryBase.md).[`execute`](QueryBase.md#execute)
***
@@ -143,7 +143,7 @@ const plan = await table.query().nearestTo([0.5, 0.2]).explainPlan();
#### Inherited from
`StandardQueryBase.explainPlan`
[`QueryBase`](QueryBase.md).[`explainPlan`](QueryBase.md#explainplan)
***
@@ -164,7 +164,7 @@ Use [Table#optimize](Table.md#optimize) to index all un-indexed data.
#### Inherited from
`StandardQueryBase.fastSearch`
[`QueryBase`](QueryBase.md).[`fastSearch`](QueryBase.md#fastsearch)
***
@@ -194,7 +194,7 @@ Use `where` instead
#### Inherited from
`StandardQueryBase.filter`
[`QueryBase`](QueryBase.md).[`filter`](QueryBase.md#filter)
***
@@ -216,7 +216,7 @@ fullTextSearch(query, options?): this
#### Inherited from
`StandardQueryBase.fullTextSearch`
[`QueryBase`](QueryBase.md).[`fullTextSearch`](QueryBase.md#fulltextsearch)
***
@@ -241,7 +241,7 @@ called then every valid row from the table will be returned.
#### Inherited from
`StandardQueryBase.limit`
[`QueryBase`](QueryBase.md).[`limit`](QueryBase.md#limit)
***
@@ -325,10 +325,6 @@ nearestToText(query, columns?): Query
offset(offset): this
```
Set the number of rows to skip before returning results.
This is useful for pagination.
#### Parameters
* **offset**: `number`
@@ -339,7 +335,7 @@ This is useful for pagination.
#### Inherited from
`StandardQueryBase.offset`
[`QueryBase`](QueryBase.md).[`offset`](QueryBase.md#offset)
***
@@ -392,7 +388,7 @@ object insertion order is easy to get wrong and `Map` is more foolproof.
#### Inherited from
`StandardQueryBase.select`
[`QueryBase`](QueryBase.md).[`select`](QueryBase.md#select)
***
@@ -414,7 +410,7 @@ Collect the results as an array of objects.
#### Inherited from
`StandardQueryBase.toArray`
[`QueryBase`](QueryBase.md).[`toArray`](QueryBase.md#toarray)
***
@@ -440,7 +436,7 @@ ArrowTable.
#### Inherited from
`StandardQueryBase.toArrow`
[`QueryBase`](QueryBase.md).[`toArrow`](QueryBase.md#toarrow)
***
@@ -475,7 +471,7 @@ on the filter column(s).
#### Inherited from
`StandardQueryBase.where`
[`QueryBase`](QueryBase.md).[`where`](QueryBase.md#where)
***
@@ -497,4 +493,4 @@ order to perform hybrid search.
#### Inherited from
`StandardQueryBase.withRowId`
[`QueryBase`](QueryBase.md).[`withRowId`](QueryBase.md#withrowid)

View File

@@ -15,11 +15,12 @@ Common methods supported by all query types
## Extended by
- [`TakeQuery`](TakeQuery.md)
- [`Query`](Query.md)
- [`VectorQuery`](VectorQuery.md)
## Type Parameters
**NativeQueryType** *extends* `NativeQuery` \| `NativeVectorQuery` \| `NativeTakeQuery`
**NativeQueryType** *extends* `NativeQuery` \| `NativeVectorQuery`
## Implements
@@ -140,6 +141,104 @@ const plan = await table.query().nearestTo([0.5, 0.2]).explainPlan();
***
### fastSearch()
```ts
fastSearch(): this
```
Skip searching un-indexed data. This can make search faster, but will miss
any data that is not yet indexed.
Use [Table#optimize](Table.md#optimize) to index all un-indexed data.
#### Returns
`this`
***
### ~~filter()~~
```ts
filter(predicate): this
```
A filter statement to be applied to this query.
#### Parameters
* **predicate**: `string`
#### Returns
`this`
#### See
where
#### Deprecated
Use `where` instead
***
### fullTextSearch()
```ts
fullTextSearch(query, options?): this
```
#### Parameters
* **query**: `string` \| [`FullTextQuery`](../interfaces/FullTextQuery.md)
* **options?**: `Partial`&lt;[`FullTextSearchOptions`](../interfaces/FullTextSearchOptions.md)&gt;
#### Returns
`this`
***
### limit()
```ts
limit(limit): this
```
Set the maximum number of results to return.
By default, a plain search has no limit. If this method is not
called then every valid row from the table will be returned.
#### Parameters
* **limit**: `number`
#### Returns
`this`
***
### offset()
```ts
offset(offset): this
```
#### Parameters
* **offset**: `number`
#### Returns
`this`
***
### select()
```ts
@@ -229,6 +328,37 @@ ArrowTable.
***
### where()
```ts
where(predicate): this
```
A filter statement to be applied to this query.
The filter should be supplied as an SQL query string. For example:
#### Parameters
* **predicate**: `string`
#### Returns
`this`
#### Example
```ts
x > 10
y > 0 AND y < 100
x > 5 OR y = 'test'
Filtering performance can often be improved by creating a scalar index
on the filter column(s).
```
***
### withRowId()
```ts

View File

@@ -9,8 +9,7 @@
A session for managing caches and object stores across LanceDB operations.
Sessions allow you to configure cache sizes for index and metadata caches,
which can significantly impact memory use and performance. They can
also be re-used across multiple connections to share the same cache state.
which can significantly impact performance for large datasets.
## Constructors
@@ -25,11 +24,8 @@ Create a new session with custom cache sizes.
# Parameters
- `index_cache_size_bytes`: The size of the index cache in bytes.
Index data is stored in memory in this cache to speed up queries.
Defaults to 6GB if not specified.
- `metadata_cache_size_bytes`: The size of the metadata cache in bytes.
The metadata cache stores file metadata and schema information in memory.
This cache improves scan and write performance.
Defaults to 1GB if not specified.
#### Parameters

View File

@@ -1,70 +0,0 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / StaticHeaderProvider
# Class: StaticHeaderProvider
Example implementation: A simple header provider that returns static headers.
This is an example implementation showing how to create a HeaderProvider
for cases where headers don't change during the session.
## Example
```typescript
const provider = new StaticHeaderProvider({
authorization: "Bearer my-token",
"X-Custom-Header": "custom-value"
});
const headers = provider.getHeaders();
// Returns: {authorization: 'Bearer my-token', 'X-Custom-Header': 'custom-value'}
```
## Extends
- [`HeaderProvider`](HeaderProvider.md)
## Constructors
### new StaticHeaderProvider()
```ts
new StaticHeaderProvider(headers): StaticHeaderProvider
```
Initialize with static headers.
#### Parameters
* **headers**: `Record`&lt;`string`, `string`&gt;
Headers to return for every request.
#### Returns
[`StaticHeaderProvider`](StaticHeaderProvider.md)
#### Overrides
[`HeaderProvider`](HeaderProvider.md).[`constructor`](HeaderProvider.md#constructors)
## Methods
### getHeaders()
```ts
getHeaders(): Record<string, string>
```
Return the static headers.
#### Returns
`Record`&lt;`string`, `string`&gt;
Copy of the static headers.
#### Overrides
[`HeaderProvider`](HeaderProvider.md).[`getHeaders`](HeaderProvider.md#getheaders)

View File

@@ -674,48 +674,6 @@ console.log(tags); // { "v1": { version: 1, manifestSize: ... } }
***
### takeOffsets()
```ts
abstract takeOffsets(offsets): TakeQuery
```
Create a query that returns a subset of the rows in the table.
#### Parameters
* **offsets**: `number`[]
The offsets of the rows to return.
#### Returns
[`TakeQuery`](TakeQuery.md)
A builder that can be used to parameterize the query.
***
### takeRowIds()
```ts
abstract takeRowIds(rowIds): TakeQuery
```
Create a query that returns a subset of the rows in the table.
#### Parameters
* **rowIds**: `number`[]
The row ids of the rows to return.
#### Returns
[`TakeQuery`](TakeQuery.md)
A builder that can be used to parameterize the query.
***
### toArrow()
```ts

View File

@@ -1,265 +0,0 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / TakeQuery
# Class: TakeQuery
A query that returns a subset of the rows in the table.
## Extends
- [`QueryBase`](QueryBase.md)&lt;`NativeTakeQuery`&gt;
## Properties
### inner
```ts
protected inner: TakeQuery | Promise<TakeQuery>;
```
#### Inherited from
[`QueryBase`](QueryBase.md).[`inner`](QueryBase.md#inner)
## Methods
### analyzePlan()
```ts
analyzePlan(): Promise<string>
```
Executes the query and returns the physical query plan annotated with runtime metrics.
This is useful for debugging and performance analysis, as it shows how the query was executed
and includes metrics such as elapsed time, rows processed, and I/O statistics.
#### Returns
`Promise`&lt;`string`&gt;
A query execution plan with runtime metrics for each step.
#### Example
```ts
import * as lancedb from "@lancedb/lancedb"
const db = await lancedb.connect("./.lancedb");
const table = await db.createTable("my_table", [
{ vector: [1.1, 0.9], id: "1" },
]);
const plan = await table.query().nearestTo([0.5, 0.2]).analyzePlan();
Example output (with runtime metrics inlined):
AnalyzeExec verbose=true, metrics=[]
ProjectionExec: expr=[id@3 as id, vector@0 as vector, _distance@2 as _distance], metrics=[output_rows=1, elapsed_compute=3.292µs]
Take: columns="vector, _rowid, _distance, (id)", metrics=[output_rows=1, elapsed_compute=66.001µs, batches_processed=1, bytes_read=8, iops=1, requests=1]
CoalesceBatchesExec: target_batch_size=1024, metrics=[output_rows=1, elapsed_compute=3.333µs]
GlobalLimitExec: skip=0, fetch=10, metrics=[output_rows=1, elapsed_compute=167ns]
FilterExec: _distance@2 IS NOT NULL, metrics=[output_rows=1, elapsed_compute=8.542µs]
SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], metrics=[output_rows=1, elapsed_compute=63.25µs, row_replacements=1]
KNNVectorDistance: metric=l2, metrics=[output_rows=1, elapsed_compute=114.333µs, output_batches=1]
LanceScan: uri=/path/to/data, projection=[vector], row_id=true, row_addr=false, ordered=false, metrics=[output_rows=1, elapsed_compute=103.626µs, bytes_read=549, iops=2, requests=2]
```
#### Inherited from
[`QueryBase`](QueryBase.md).[`analyzePlan`](QueryBase.md#analyzeplan)
***
### execute()
```ts
protected execute(options?): RecordBatchIterator
```
Execute the query and return the results as an
#### Parameters
* **options?**: `Partial`&lt;[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)&gt;
#### Returns
[`RecordBatchIterator`](RecordBatchIterator.md)
#### See
- AsyncIterator
of
- RecordBatch.
By default, LanceDb will use many threads to calculate results and, when
the result set is large, multiple batches will be processed at one time.
This readahead is limited however and backpressure will be applied if this
stream is consumed slowly (this constrains the maximum memory used by a
single query)
#### Inherited from
[`QueryBase`](QueryBase.md).[`execute`](QueryBase.md#execute)
***
### explainPlan()
```ts
explainPlan(verbose): Promise<string>
```
Generates an explanation of the query execution plan.
#### Parameters
* **verbose**: `boolean` = `false`
If true, provides a more detailed explanation. Defaults to false.
#### Returns
`Promise`&lt;`string`&gt;
A Promise that resolves to a string containing the query execution plan explanation.
#### Example
```ts
import * as lancedb from "@lancedb/lancedb"
const db = await lancedb.connect("./.lancedb");
const table = await db.createTable("my_table", [
{ vector: [1.1, 0.9], id: "1" },
]);
const plan = await table.query().nearestTo([0.5, 0.2]).explainPlan();
```
#### Inherited from
[`QueryBase`](QueryBase.md).[`explainPlan`](QueryBase.md#explainplan)
***
### select()
```ts
select(columns): this
```
Return only the specified columns.
By default a query will return all columns from the table. However, this can have
a very significant impact on latency. LanceDb stores data in a columnar fashion. This
means we can finely tune our I/O to select exactly the columns we need.
As a best practice you should always limit queries to the columns that you need. If you
pass in an array of column names then only those columns will be returned.
You can also use this method to create new "dynamic" columns based on your existing columns.
For example, you may not care about "a" or "b" but instead simply want "a + b". This is often
seen in the SELECT clause of an SQL query (e.g. `SELECT a+b FROM my_table`).
To create dynamic columns you can pass in a Map<string, string>. A column will be returned
for each entry in the map. The key provides the name of the column. The value is
an SQL string used to specify how the column is calculated.
For example, an SQL query might state `SELECT a + b AS combined, c`. The equivalent
input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
#### Returns
`this`
#### Example
```ts
new Map([["combined", "a + b"], ["c", "c"]])
Columns will always be returned in the order given, even if that order is different than
the order used when adding the data.
Note that you can pass in a `Record<string, string>` (e.g. an object literal). This method
uses `Object.entries` which should preserve the insertion order of the object. However,
object insertion order is easy to get wrong and `Map` is more foolproof.
```
#### Inherited from
[`QueryBase`](QueryBase.md).[`select`](QueryBase.md#select)
***
### toArray()
```ts
toArray(options?): Promise<any[]>
```
Collect the results as an array of objects.
#### Parameters
* **options?**: `Partial`&lt;[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)&gt;
#### Returns
`Promise`&lt;`any`[]&gt;
#### Inherited from
[`QueryBase`](QueryBase.md).[`toArray`](QueryBase.md#toarray)
***
### toArrow()
```ts
toArrow(options?): Promise<Table<any>>
```
Collect the results as an Arrow
#### Parameters
* **options?**: `Partial`&lt;[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)&gt;
#### Returns
`Promise`&lt;`Table`&lt;`any`&gt;&gt;
#### See
ArrowTable.
#### Inherited from
[`QueryBase`](QueryBase.md).[`toArrow`](QueryBase.md#toarrow)
***
### withRowId()
```ts
withRowId(): this
```
Whether to return the row id in the results.
This column can be used to match results between different queries. For
example, to match results from a full text search and a vector search in
order to perform hybrid search.
#### Returns
`this`
#### Inherited from
[`QueryBase`](QueryBase.md).[`withRowId`](QueryBase.md#withrowid)

View File

@@ -16,7 +16,7 @@ This builder can be reused to execute the query many times.
## Extends
- `StandardQueryBase`&lt;`NativeVectorQuery`&gt;
- [`QueryBase`](QueryBase.md)&lt;`NativeVectorQuery`&gt;
## Properties
@@ -28,7 +28,7 @@ protected inner: VectorQuery | Promise<VectorQuery>;
#### Inherited from
`StandardQueryBase.inner`
[`QueryBase`](QueryBase.md).[`inner`](QueryBase.md#inner)
## Methods
@@ -91,7 +91,7 @@ AnalyzeExec verbose=true, metrics=[]
#### Inherited from
`StandardQueryBase.analyzePlan`
[`QueryBase`](QueryBase.md).[`analyzePlan`](QueryBase.md#analyzeplan)
***
@@ -248,7 +248,7 @@ single query)
#### Inherited from
`StandardQueryBase.execute`
[`QueryBase`](QueryBase.md).[`execute`](QueryBase.md#execute)
***
@@ -284,7 +284,7 @@ const plan = await table.query().nearestTo([0.5, 0.2]).explainPlan();
#### Inherited from
`StandardQueryBase.explainPlan`
[`QueryBase`](QueryBase.md).[`explainPlan`](QueryBase.md#explainplan)
***
@@ -305,7 +305,7 @@ Use [Table#optimize](Table.md#optimize) to index all un-indexed data.
#### Inherited from
`StandardQueryBase.fastSearch`
[`QueryBase`](QueryBase.md).[`fastSearch`](QueryBase.md#fastsearch)
***
@@ -335,7 +335,7 @@ Use `where` instead
#### Inherited from
`StandardQueryBase.filter`
[`QueryBase`](QueryBase.md).[`filter`](QueryBase.md#filter)
***
@@ -357,7 +357,7 @@ fullTextSearch(query, options?): this
#### Inherited from
`StandardQueryBase.fullTextSearch`
[`QueryBase`](QueryBase.md).[`fullTextSearch`](QueryBase.md#fulltextsearch)
***
@@ -382,7 +382,7 @@ called then every valid row from the table will be returned.
#### Inherited from
`StandardQueryBase.limit`
[`QueryBase`](QueryBase.md).[`limit`](QueryBase.md#limit)
***
@@ -480,10 +480,6 @@ the minimum and maximum to the same value.
offset(offset): this
```
Set the number of rows to skip before returning results.
This is useful for pagination.
#### Parameters
* **offset**: `number`
@@ -494,7 +490,7 @@ This is useful for pagination.
#### Inherited from
`StandardQueryBase.offset`
[`QueryBase`](QueryBase.md).[`offset`](QueryBase.md#offset)
***
@@ -641,7 +637,7 @@ object insertion order is easy to get wrong and `Map` is more foolproof.
#### Inherited from
`StandardQueryBase.select`
[`QueryBase`](QueryBase.md).[`select`](QueryBase.md#select)
***
@@ -663,7 +659,7 @@ Collect the results as an array of objects.
#### Inherited from
`StandardQueryBase.toArray`
[`QueryBase`](QueryBase.md).[`toArray`](QueryBase.md#toarray)
***
@@ -689,7 +685,7 @@ ArrowTable.
#### Inherited from
`StandardQueryBase.toArrow`
[`QueryBase`](QueryBase.md).[`toArrow`](QueryBase.md#toarrow)
***
@@ -724,7 +720,7 @@ on the filter column(s).
#### Inherited from
`StandardQueryBase.where`
[`QueryBase`](QueryBase.md).[`where`](QueryBase.md#where)
***
@@ -746,4 +742,4 @@ order to perform hybrid search.
#### Inherited from
`StandardQueryBase.withRowId`
[`QueryBase`](QueryBase.md).[`withRowId`](QueryBase.md#withrowid)

View File

@@ -6,14 +6,13 @@
# Function: connect()
## connect(uri, options, session, headerProvider)
## connect(uri, options, session)
```ts
function connect(
uri,
options?,
session?,
headerProvider?): Promise<Connection>
session?): Promise<Connection>
```
Connect to a LanceDB instance at the given URI.
@@ -35,8 +34,6 @@ Accepted formats:
* **session?**: [`Session`](../classes/Session.md)
* **headerProvider?**: [`HeaderProvider`](../classes/HeaderProvider.md) \| () => `Record`&lt;`string`, `string`&gt; \| () => `Promise`&lt;`Record`&lt;`string`, `string`&gt;&gt;
### Returns
`Promise`&lt;[`Connection`](../classes/Connection.md)&gt;
@@ -58,18 +55,6 @@ const conn = await connect(
});
```
Using with a header provider for per-request authentication:
```ts
const provider = new StaticHeaderProvider({
"X-API-Key": "my-key"
});
const conn = await connectWithHeaderProvider(
"db://host:port",
options,
provider
);
```
## connect(options)
```ts

View File

@@ -20,24 +20,19 @@
- [BooleanQuery](classes/BooleanQuery.md)
- [BoostQuery](classes/BoostQuery.md)
- [Connection](classes/Connection.md)
- [HeaderProvider](classes/HeaderProvider.md)
- [Index](classes/Index.md)
- [MakeArrowTableOptions](classes/MakeArrowTableOptions.md)
- [MatchQuery](classes/MatchQuery.md)
- [MergeInsertBuilder](classes/MergeInsertBuilder.md)
- [MultiMatchQuery](classes/MultiMatchQuery.md)
- [NativeJsHeaderProvider](classes/NativeJsHeaderProvider.md)
- [OAuthHeaderProvider](classes/OAuthHeaderProvider.md)
- [PhraseQuery](classes/PhraseQuery.md)
- [Query](classes/Query.md)
- [QueryBase](classes/QueryBase.md)
- [RecordBatchIterator](classes/RecordBatchIterator.md)
- [Session](classes/Session.md)
- [StaticHeaderProvider](classes/StaticHeaderProvider.md)
- [Table](classes/Table.md)
- [TagContents](classes/TagContents.md)
- [Tags](classes/Tags.md)
- [TakeQuery](classes/TakeQuery.md)
- [VectorColumnOptions](classes/VectorColumnOptions.md)
- [VectorQuery](classes/VectorQuery.md)
@@ -78,7 +73,6 @@
- [TableNamesOptions](interfaces/TableNamesOptions.md)
- [TableStatistics](interfaces/TableStatistics.md)
- [TimeoutConfig](interfaces/TimeoutConfig.md)
- [TokenResponse](interfaces/TokenResponse.md)
- [UpdateOptions](interfaces/UpdateOptions.md)
- [UpdateResult](interfaces/UpdateResult.md)
- [Version](interfaces/Version.md)

View File

@@ -16,14 +16,6 @@ optional extraHeaders: Record<string, string>;
***
### idDelimiter?
```ts
optional idDelimiter: string;
```
***
### retryConfig?
```ts

View File

@@ -26,18 +26,6 @@ will be used to determine the most useful kind of index to create.
***
### name?
```ts
optional name: string;
```
Optional custom name for the index.
If not provided, a default name will be generated based on the column name.
***
### replace?
```ts
@@ -54,27 +42,8 @@ The default is true
***
### train?
```ts
optional train: boolean;
```
Whether to train the index with existing data.
If true (default), the index will be trained with existing data in the table.
If false, the index will be created empty and populated as new data is added.
Note: This option is only supported for scalar indices. Vector indices always train.
***
### waitTimeoutSeconds?
```ts
optional waitTimeoutSeconds: number;
```
Timeout in seconds to wait for index creation to complete.
If not specified, the method will return immediately after starting the index creation.

View File

@@ -44,17 +44,3 @@ optional readTimeout: number;
The timeout for reading data from the server in seconds. Default is 300
seconds (5 minutes). This can also be set via the environment variable
`LANCE_CLIENT_READ_TIMEOUT`, as an integer number of seconds.
***
### timeout?
```ts
optional timeout: number;
```
The overall timeout for the entire request in seconds. This includes
connection, send, and read time. If the entire request doesn't complete
within this time, it will fail. Default is None (no overall timeout).
This can also be set via the environment variable `LANCE_CLIENT_TIMEOUT`,
as an integer number of seconds.

View File

@@ -1,25 +0,0 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / TokenResponse
# Interface: TokenResponse
Token response from OAuth provider.
## Properties
### accessToken
```ts
accessToken: string;
```
***
### expiresIn?
```ts
optional expiresIn: number;
```

View File

@@ -15,7 +15,7 @@ publish = false
crate-type = ["cdylib"]
[dependencies]
lancedb = { path = "../../../rust/lancedb", default-features = false }
lancedb = { path = "../../../rust/lancedb" }
lance = { workspace = true }
arrow = { workspace = true, features = ["ffi"] }
arrow-schema.workspace = true
@@ -25,6 +25,3 @@ snafu.workspace = true
lazy_static.workspace = true
serde = { version = "^1" }
serde_json = { version = "1" }
[features]
default = ["lancedb/default"]

View File

@@ -16,7 +16,6 @@ pub trait JNIEnvExt {
fn get_integers(&mut self, obj: &JObject) -> Result<Vec<i32>>;
/// Get strings from Java List<String> object.
#[allow(dead_code)]
fn get_strings(&mut self, obj: &JObject) -> Result<Vec<String>>;
/// Get strings from Java String[] object.

View File

@@ -6,7 +6,6 @@ use jni::JNIEnv;
use crate::Result;
#[allow(dead_code)]
pub trait FromJObject<T> {
fn extract(&self) -> Result<T>;
}
@@ -40,7 +39,6 @@ impl FromJObject<f64> for JObject<'_> {
}
}
#[allow(dead_code)]
pub trait FromJString {
fn extract(&self, env: &mut JNIEnv) -> Result<String>;
}
@@ -68,7 +66,6 @@ pub trait JMapExt {
fn get_f64(&self, env: &mut JNIEnv, key: &str) -> Result<Option<f64>>;
}
#[allow(dead_code)]
fn get_map_value<T>(env: &mut JNIEnv, map: &JMap, key: &str) -> Result<Option<T>>
where
for<'a> JObject<'a>: FromJObject<T>,

View File

@@ -8,7 +8,7 @@
<parent>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.22.1-beta.3</version>
<version>0.21.2-final.0</version>
<relativePath>../pom.xml</relativePath>
</parent>

View File

@@ -8,7 +8,7 @@
<parent>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.22.1-beta.3</version>
<version>0.21.2-final.0</version>
<relativePath>../pom.xml</relativePath>
</parent>

View File

@@ -6,7 +6,7 @@
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.22.1-beta.3</version>
<version>0.21.2-final.0</version>
<packaging>pom</packaging>
<name>${project.artifactId}</name>
<description>LanceDB Java SDK Parent POM</description>

View File

@@ -1,7 +1,7 @@
[package]
name = "lancedb-nodejs"
edition.workspace = true
version = "0.22.1-beta.3"
version = "0.21.2"
license.workspace = true
description.workspace = true
repository.workspace = true
@@ -18,7 +18,7 @@ arrow-array.workspace = true
arrow-schema.workspace = true
env_logger.workspace = true
futures.workspace = true
lancedb = { path = "../rust/lancedb", default-features = false }
lancedb = { path = "../rust/lancedb" }
napi = { version = "2.16.8", default-features = false, features = [
"napi9",
"async"
@@ -36,6 +36,6 @@ aws-lc-rs = "=1.13.0"
napi-build = "2.1"
[features]
default = ["remote", "lancedb/default"]
default = ["remote"]
fp16kernels = ["lancedb/fp16kernels"]
remote = ["lancedb/remote"]

View File

@@ -1008,64 +1008,5 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
expect(result).toEqual(null);
});
});
describe("boolean null handling", function () {
it("should handle null values in nullable boolean fields", () => {
const { makeArrowTable } = require("../lancedb/arrow");
const schema = new Schema([new Field("test", new arrow.Bool(), true)]);
// Test with all null values
const data = [{ test: null }];
const table = makeArrowTable(data, { schema });
expect(table.numRows).toBe(1);
expect(table.schema.names).toEqual(["test"]);
expect(table.getChild("test")!.get(0)).toBeNull();
});
it("should handle mixed null and non-null boolean values", () => {
const { makeArrowTable } = require("../lancedb/arrow");
const schema = new Schema([new Field("test", new Bool(), true)]);
// Test with mixed values
const data = [{ test: true }, { test: null }, { test: false }];
const table = makeArrowTable(data, { schema });
expect(table.numRows).toBe(3);
expect(table.getChild("test")!.get(0)).toBe(true);
expect(table.getChild("test")!.get(1)).toBeNull();
expect(table.getChild("test")!.get(2)).toBe(false);
});
});
},
);
// Test for the undefined values bug fix
describe("undefined values handling", () => {
it("should handle mixed undefined and actual values", () => {
const schema = new Schema([
new Field("text", new Utf8(), true), // nullable
new Field("number", new Int32(), true), // nullable
new Field("bool", new Bool(), true), // nullable
]);
const data = [
{ text: undefined, number: 42, bool: true },
{ text: "hello", number: undefined, bool: false },
{ text: "world", number: 123, bool: undefined },
];
const table = makeArrowTable(data, { schema });
const result = table.toArray();
expect(result).toHaveLength(3);
expect(result[0].text).toBe(null);
expect(result[0].number).toBe(42);
expect(result[0].bool).toBe(true);
expect(result[1].text).toBe("hello");
expect(result[1].number).toBe(null);
expect(result[1].bool).toBe(false);
expect(result[2].text).toBe("world");
expect(result[2].number).toBe(123);
expect(result[2].bool).toBe(null);
});
});

View File

@@ -203,106 +203,3 @@ describe("given a connection", () => {
});
});
});
describe("clone table functionality", () => {
let tmpDir: tmp.DirResult;
let db: Connection;
beforeEach(async () => {
tmpDir = tmp.dirSync({ unsafeCleanup: true });
db = await connect(tmpDir.name);
});
afterEach(() => tmpDir.removeCallback());
it("should clone a table with latest version (default behavior)", async () => {
// Create source table with some data
const data = [
{ id: 1, text: "hello", vector: [1.0, 2.0] },
{ id: 2, text: "world", vector: [3.0, 4.0] },
];
const sourceTable = await db.createTable("source", data);
// Add more data to create a new version
const moreData = [{ id: 3, text: "test", vector: [5.0, 6.0] }];
await sourceTable.add(moreData);
// Clone the table (should get latest version with 3 rows)
const sourceUri = `${tmpDir.name}/source.lance`;
const clonedTable = await db.cloneTable("cloned", sourceUri);
// Verify cloned table has all 3 rows
expect(await clonedTable.countRows()).toBe(3);
expect((await db.tableNames()).includes("cloned")).toBe(true);
});
it("should clone a table from a specific version", async () => {
// Create source table with initial data
const data = [
{ id: 1, text: "hello", vector: [1.0, 2.0] },
{ id: 2, text: "world", vector: [3.0, 4.0] },
];
const sourceTable = await db.createTable("source", data);
// Get the initial version
const initialVersion = await sourceTable.version();
// Add more data to create a new version
const moreData = [{ id: 3, text: "test", vector: [5.0, 6.0] }];
await sourceTable.add(moreData);
// Verify source now has 3 rows
expect(await sourceTable.countRows()).toBe(3);
// Clone from the initial version (should have only 2 rows)
const sourceUri = `${tmpDir.name}/source.lance`;
const clonedTable = await db.cloneTable("cloned", sourceUri, {
sourceVersion: initialVersion,
});
// Verify cloned table has only the initial 2 rows
expect(await clonedTable.countRows()).toBe(2);
});
it("should clone a table from a tagged version", async () => {
// Create source table with initial data
const data = [
{ id: 1, text: "hello", vector: [1.0, 2.0] },
{ id: 2, text: "world", vector: [3.0, 4.0] },
];
const sourceTable = await db.createTable("source", data);
// Create a tag for the current version
const tags = await sourceTable.tags();
await tags.create("v1.0", await sourceTable.version());
// Add more data after the tag
const moreData = [{ id: 3, text: "test", vector: [5.0, 6.0] }];
await sourceTable.add(moreData);
// Verify source now has 3 rows
expect(await sourceTable.countRows()).toBe(3);
// Clone from the tagged version (should have only 2 rows)
const sourceUri = `${tmpDir.name}/source.lance`;
const clonedTable = await db.cloneTable("cloned", sourceUri, {
sourceTag: "v1.0",
});
// Verify cloned table has only the tagged version's 2 rows
expect(await clonedTable.countRows()).toBe(2);
});
it("should fail when attempting deep clone", async () => {
// Create source table with some data
const data = [
{ id: 1, text: "hello", vector: [1.0, 2.0] },
{ id: 2, text: "world", vector: [3.0, 4.0] },
];
await db.createTable("source", data);
// Try to create a deep clone (should fail)
const sourceUri = `${tmpDir.name}/source.lance`;
await expect(
db.cloneTable("cloned", sourceUri, { isShallow: false }),
).rejects.toThrow("Deep clone is not yet implemented");
});
});

View File

@@ -256,60 +256,6 @@ describe("embedding functions", () => {
expect(actual).toHaveProperty("text");
});
it("should handle undefined vector field with embedding function correctly", async () => {
@register("undefined_test")
class MockEmbeddingFunction extends EmbeddingFunction<string> {
ndims() {
return 3;
}
embeddingDataType(): Float {
return new Float32();
}
async computeQueryEmbeddings(_data: string) {
return [1, 2, 3];
}
async computeSourceEmbeddings(data: string[]) {
return Array.from({ length: data.length }).fill([
1, 2, 3,
]) as number[][];
}
}
const func = getRegistry()
.get<MockEmbeddingFunction>("undefined_test")!
.create();
const schema = new Schema([
new Field("text", new Utf8(), true),
new Field(
"vector",
new FixedSizeList(3, new Field("item", new Float32(), true)),
true,
),
]);
const db = await connect(tmpDir.name);
const table = await db.createEmptyTable("test_undefined", schema, {
embeddingFunction: {
function: func,
sourceColumn: "text",
vectorColumn: "vector",
},
});
// Test that undefined, null, and omitted vector fields all work
await table.add([{ text: "test1", vector: undefined }]);
await table.add([{ text: "test2", vector: null }]);
await table.add([{ text: "test3" }]);
const rows = await table.query().toArray();
expect(rows.length).toBe(3);
// All rows should have vectors computed by the embedding function
for (const row of rows) {
expect(row.vector).toBeDefined();
expect(JSON.parse(JSON.stringify(row.vector))).toEqual([1, 2, 3]);
}
});
test.each([new Float16(), new Float32(), new Float64()])(
"should be able to provide manual embeddings with multiple float datatype",
async (floatType) => {

View File

@@ -3,50 +3,7 @@
import * as http from "http";
import { RequestListener } from "http";
import {
ClientConfig,
Connection,
ConnectionOptions,
NativeJsHeaderProvider,
TlsConfig,
connect,
} from "../lancedb";
import {
HeaderProvider,
OAuthHeaderProvider,
StaticHeaderProvider,
} from "../lancedb/header";
// Test-only header providers
class CustomProvider extends HeaderProvider {
getHeaders(): Record<string, string> {
return { "X-Custom": "custom-value" };
}
}
class ErrorProvider extends HeaderProvider {
private errorMessage: string;
public callCount: number = 0;
constructor(errorMessage: string = "Test error") {
super();
this.errorMessage = errorMessage;
}
getHeaders(): Record<string, string> {
this.callCount++;
throw new Error(this.errorMessage);
}
}
class ConcurrentProvider extends HeaderProvider {
private counter: number = 0;
getHeaders(): Record<string, string> {
this.counter++;
return { "X-Request-Id": String(this.counter) };
}
}
import { Connection, ConnectionOptions, connect } from "../lancedb";
async function withMockDatabase(
listener: RequestListener,
@@ -191,431 +148,4 @@ describe("remote connection", () => {
},
);
});
describe("TlsConfig", () => {
it("should create TlsConfig with all fields", () => {
const tlsConfig: TlsConfig = {
certFile: "/path/to/cert.pem",
keyFile: "/path/to/key.pem",
sslCaCert: "/path/to/ca.pem",
assertHostname: false,
};
expect(tlsConfig.certFile).toBe("/path/to/cert.pem");
expect(tlsConfig.keyFile).toBe("/path/to/key.pem");
expect(tlsConfig.sslCaCert).toBe("/path/to/ca.pem");
expect(tlsConfig.assertHostname).toBe(false);
});
it("should create TlsConfig with partial fields", () => {
const tlsConfig: TlsConfig = {
certFile: "/path/to/cert.pem",
keyFile: "/path/to/key.pem",
};
expect(tlsConfig.certFile).toBe("/path/to/cert.pem");
expect(tlsConfig.keyFile).toBe("/path/to/key.pem");
expect(tlsConfig.sslCaCert).toBeUndefined();
expect(tlsConfig.assertHostname).toBeUndefined();
});
it("should create ClientConfig with TlsConfig", () => {
const tlsConfig: TlsConfig = {
certFile: "/path/to/cert.pem",
keyFile: "/path/to/key.pem",
sslCaCert: "/path/to/ca.pem",
assertHostname: true,
};
const clientConfig: ClientConfig = {
userAgent: "test-agent",
tlsConfig: tlsConfig,
};
expect(clientConfig.userAgent).toBe("test-agent");
expect(clientConfig.tlsConfig).toBeDefined();
expect(clientConfig.tlsConfig?.certFile).toBe("/path/to/cert.pem");
expect(clientConfig.tlsConfig?.keyFile).toBe("/path/to/key.pem");
expect(clientConfig.tlsConfig?.sslCaCert).toBe("/path/to/ca.pem");
expect(clientConfig.tlsConfig?.assertHostname).toBe(true);
});
it("should handle empty TlsConfig", () => {
const tlsConfig: TlsConfig = {};
expect(tlsConfig.certFile).toBeUndefined();
expect(tlsConfig.keyFile).toBeUndefined();
expect(tlsConfig.sslCaCert).toBeUndefined();
expect(tlsConfig.assertHostname).toBeUndefined();
});
it("should accept TlsConfig in connection options", () => {
const tlsConfig: TlsConfig = {
certFile: "/path/to/cert.pem",
keyFile: "/path/to/key.pem",
sslCaCert: "/path/to/ca.pem",
assertHostname: false,
};
// Just verify that the ClientConfig accepts the TlsConfig
const clientConfig: ClientConfig = {
tlsConfig: tlsConfig,
};
const connectionOptions: ConnectionOptions = {
apiKey: "fake",
clientConfig: clientConfig,
};
// Verify the configuration structure is correct
expect(connectionOptions.clientConfig).toBeDefined();
expect(connectionOptions.clientConfig?.tlsConfig).toBeDefined();
expect(connectionOptions.clientConfig?.tlsConfig?.certFile).toBe(
"/path/to/cert.pem",
);
});
});
describe("header providers", () => {
it("should work with StaticHeaderProvider", async () => {
const provider = new StaticHeaderProvider({
authorization: "Bearer test-token",
"X-Custom": "value",
});
const headers = provider.getHeaders();
expect(headers).toEqual({
authorization: "Bearer test-token",
"X-Custom": "value",
});
// Test that it returns a copy
headers["X-Modified"] = "modified";
const headers2 = provider.getHeaders();
expect(headers2).not.toHaveProperty("X-Modified");
});
it("should pass headers from StaticHeaderProvider to requests", async () => {
const provider = new StaticHeaderProvider({
"X-Custom-Auth": "secret-token",
"X-Request-Source": "test-suite",
});
await withMockDatabase(
(req, res) => {
expect(req.headers["x-custom-auth"]).toEqual("secret-token");
expect(req.headers["x-request-source"]).toEqual("test-suite");
const body = JSON.stringify({ tables: [] });
res.writeHead(200, { "Content-Type": "application/json" }).end(body);
},
async () => {
// Use actual header provider mechanism instead of extraHeaders
const conn = await connect(
"db://dev",
{
apiKey: "fake",
hostOverride: "http://localhost:8000",
},
undefined, // session
provider, // headerProvider
);
const tableNames = await conn.tableNames();
expect(tableNames).toEqual([]);
},
);
});
it("should work with CustomProvider", () => {
const provider = new CustomProvider();
const headers = provider.getHeaders();
expect(headers).toEqual({ "X-Custom": "custom-value" });
});
it("should handle ErrorProvider errors", () => {
const provider = new ErrorProvider("Authentication failed");
expect(() => provider.getHeaders()).toThrow("Authentication failed");
expect(provider.callCount).toBe(1);
// Test that error is thrown each time
expect(() => provider.getHeaders()).toThrow("Authentication failed");
expect(provider.callCount).toBe(2);
});
it("should work with ConcurrentProvider", () => {
const provider = new ConcurrentProvider();
const headers1 = provider.getHeaders();
const headers2 = provider.getHeaders();
const headers3 = provider.getHeaders();
expect(headers1).toEqual({ "X-Request-Id": "1" });
expect(headers2).toEqual({ "X-Request-Id": "2" });
expect(headers3).toEqual({ "X-Request-Id": "3" });
});
describe("OAuthHeaderProvider", () => {
it("should initialize correctly", () => {
const fetcher = () => ({
accessToken: "token123",
expiresIn: 3600,
});
const provider = new OAuthHeaderProvider(fetcher);
expect(provider).toBeInstanceOf(HeaderProvider);
});
it("should fetch token on first use", async () => {
let callCount = 0;
const fetcher = () => {
callCount++;
return {
accessToken: "token123",
expiresIn: 3600,
};
};
const provider = new OAuthHeaderProvider(fetcher);
// Need to manually refresh first due to sync limitation
await provider.refreshToken();
const headers = provider.getHeaders();
expect(headers).toEqual({ authorization: "Bearer token123" });
expect(callCount).toBe(1);
// Second call should not fetch again
const headers2 = provider.getHeaders();
expect(headers2).toEqual({ authorization: "Bearer token123" });
expect(callCount).toBe(1);
});
it("should handle tokens without expiry", async () => {
const fetcher = () => ({
accessToken: "permanent_token",
});
const provider = new OAuthHeaderProvider(fetcher);
await provider.refreshToken();
const headers = provider.getHeaders();
expect(headers).toEqual({ authorization: "Bearer permanent_token" });
});
it("should throw error when access_token is missing", async () => {
const fetcher = () =>
({
expiresIn: 3600,
}) as { accessToken?: string; expiresIn?: number };
const provider = new OAuthHeaderProvider(
fetcher as () => {
accessToken: string;
expiresIn?: number;
},
);
await expect(provider.refreshToken()).rejects.toThrow(
"Token fetcher did not return 'accessToken'",
);
});
it("should handle async token fetchers", async () => {
const fetcher = async () => {
// Simulate async operation
await new Promise((resolve) => setTimeout(resolve, 10));
return {
accessToken: "async_token",
expiresIn: 3600,
};
};
const provider = new OAuthHeaderProvider(fetcher);
await provider.refreshToken();
const headers = provider.getHeaders();
expect(headers).toEqual({ authorization: "Bearer async_token" });
});
});
it("should merge header provider headers with extra headers", async () => {
const provider = new StaticHeaderProvider({
"X-From-Provider": "provider-value",
});
await withMockDatabase(
(req, res) => {
expect(req.headers["x-from-provider"]).toEqual("provider-value");
expect(req.headers["x-extra-header"]).toEqual("extra-value");
const body = JSON.stringify({ tables: [] });
res.writeHead(200, { "Content-Type": "application/json" }).end(body);
},
async () => {
// Use header provider with additional extraHeaders
const conn = await connect(
"db://dev",
{
apiKey: "fake",
hostOverride: "http://localhost:8000",
clientConfig: {
extraHeaders: {
"X-Extra-Header": "extra-value",
},
},
},
undefined, // session
provider, // headerProvider
);
const tableNames = await conn.tableNames();
expect(tableNames).toEqual([]);
},
);
});
});
describe("header provider integration", () => {
it("should work with TypeScript StaticHeaderProvider", async () => {
let requestCount = 0;
await withMockDatabase(
(req, res) => {
requestCount++;
// Check headers are present on each request
expect(req.headers["authorization"]).toEqual("Bearer test-token-123");
expect(req.headers["x-custom"]).toEqual("custom-value");
// Return different responses based on the endpoint
if (req.url === "/v1/table/test_table/describe/") {
const body = JSON.stringify({
name: "test_table",
schema: { fields: [] },
});
res
.writeHead(200, { "Content-Type": "application/json" })
.end(body);
} else {
const body = JSON.stringify({ tables: ["test_table"] });
res
.writeHead(200, { "Content-Type": "application/json" })
.end(body);
}
},
async () => {
// Create provider with static headers
const provider = new StaticHeaderProvider({
authorization: "Bearer test-token-123",
"X-Custom": "custom-value",
});
// Connect with the provider
const conn = await connect(
"db://dev",
{
apiKey: "fake",
hostOverride: "http://localhost:8000",
},
undefined, // session
provider, // headerProvider
);
// Make multiple requests to verify headers are sent each time
const tables1 = await conn.tableNames();
expect(tables1).toEqual(["test_table"]);
const tables2 = await conn.tableNames();
expect(tables2).toEqual(["test_table"]);
// Verify headers were sent with each request
expect(requestCount).toBeGreaterThanOrEqual(2);
},
);
});
it("should work with JavaScript function provider", async () => {
let requestId = 0;
await withMockDatabase(
(req, res) => {
// Check dynamic header is present
expect(req.headers["x-request-id"]).toBeDefined();
expect(req.headers["x-request-id"]).toMatch(/^req-\d+$/);
const body = JSON.stringify({ tables: [] });
res.writeHead(200, { "Content-Type": "application/json" }).end(body);
},
async () => {
// Create a JavaScript function that returns dynamic headers
const getHeaders = async () => {
requestId++;
return {
"X-Request-Id": `req-${requestId}`,
"X-Timestamp": new Date().toISOString(),
};
};
// Connect with the function directly
const conn = await connect(
"db://dev",
{
apiKey: "fake",
hostOverride: "http://localhost:8000",
},
undefined, // session
getHeaders, // headerProvider
);
// Make requests - each should have different headers
const tables = await conn.tableNames();
expect(tables).toEqual([]);
},
);
});
it("should support OAuth-like token refresh pattern", async () => {
let tokenVersion = 0;
await withMockDatabase(
(req, res) => {
// Verify authorization header
const authHeader = req.headers["authorization"];
expect(authHeader).toBeDefined();
expect(authHeader).toMatch(/^Bearer token-v\d+$/);
const body = JSON.stringify({ tables: [] });
res.writeHead(200, { "Content-Type": "application/json" }).end(body);
},
async () => {
// Simulate OAuth token fetcher
const fetchToken = async () => {
tokenVersion++;
return {
authorization: `Bearer token-v${tokenVersion}`,
};
};
// Connect with the function directly
const conn = await connect(
"db://dev",
{
apiKey: "fake",
hostOverride: "http://localhost:8000",
},
undefined, // session
fetchToken, // headerProvider
);
// Each request will fetch a new token
await conn.tableNames();
// Token should be different on next request
await conn.tableNames();
},
);
});
});
});

View File

@@ -287,12 +287,6 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
expect(res2[1].id).toEqual(data2.id);
});
it("should support take queries", async () => {
await table.add([{ id: 1 }, { id: 2 }, { id: 3 }]);
const res = await table.takeOffsets([1, 2]).toArrow();
expect(res.getChild("id")?.toJSON()).toEqual([2, 3]);
});
it("should return the table as an instance of an arrow table", async () => {
const arrowTbl = await table.toArrow();
expect(arrowTbl).toBeInstanceOf(ArrowTable);
@@ -563,7 +557,7 @@ describe("When creating an index", () => {
// test offset
rst = await tbl.query().limit(2).offset(1).nearestTo(queryVec).toArrow();
expect(rst.numRows).toBe(2);
expect(rst.numRows).toBe(1);
// test nprobes
rst = await tbl.query().nearestTo(queryVec).limit(2).nprobes(50).toArrow();
@@ -702,7 +696,7 @@ describe("When creating an index", () => {
// test offset
rst = await tbl.query().limit(2).offset(1).nearestTo(queryVec).toArrow();
expect(rst.numRows).toBe(2);
expect(rst.numRows).toBe(1);
// test ef
rst = await tbl.query().limit(2).nearestTo(queryVec).ef(100).toArrow();
@@ -857,40 +851,6 @@ describe("When creating an index", () => {
expect(stats).toBeUndefined();
});
test("should support name and train parameters", async () => {
// Test with custom name
await tbl.createIndex("vec", {
config: Index.ivfPq({ numPartitions: 4 }),
name: "my_custom_vector_index",
});
const indices = await tbl.listIndices();
expect(indices).toHaveLength(1);
expect(indices[0].name).toBe("my_custom_vector_index");
// Test scalar index with train=false
await tbl.createIndex("id", {
config: Index.btree(),
name: "btree_empty",
train: false,
});
const allIndices = await tbl.listIndices();
expect(allIndices).toHaveLength(2);
expect(allIndices.some((idx) => idx.name === "btree_empty")).toBe(true);
// Test with both name and train=true (use tags column)
await tbl.createIndex("tags", {
config: Index.labelList(),
name: "tags_trained",
train: true,
});
const finalIndices = await tbl.listIndices();
expect(finalIndices).toHaveLength(3);
expect(finalIndices.some((idx) => idx.name === "tags_trained")).toBe(true);
});
test("create ivf_flat with binary vectors", async () => {
const db = await connect(tmpDir.name);
const binarySchema = new Schema([

View File

@@ -12,7 +12,7 @@ test("ann index examples", async () => {
// --8<-- [start:ingest]
const db = await lancedb.connect(databaseDir);
const data = Array.from({ length: 1_000 }, (_, i) => ({
const data = Array.from({ length: 5_000 }, (_, i) => ({
vector: Array(128).fill(i),
id: `${i}`,
content: "",
@@ -24,8 +24,8 @@ test("ann index examples", async () => {
});
await table.createIndex("vector", {
config: lancedb.Index.ivfPq({
numPartitions: 30,
numSubVectors: 8,
numPartitions: 10,
numSubVectors: 16,
}),
});
// --8<-- [end:ingest]

View File

@@ -512,11 +512,7 @@ function* rowPathsAndValues(
if (isObject(value)) {
yield* rowPathsAndValues(value, [...basePath, key]);
} else {
// Skip undefined values - they should be treated the same as missing fields
// for embedding function purposes
if (value !== undefined) {
yield [[...basePath, key], value];
}
yield [[...basePath, key], value];
}
}
}
@@ -705,7 +701,7 @@ function transposeData(
}
return current;
});
return makeVector(values, field.type, undefined, field.nullable);
return makeVector(values, field.type);
}
}
@@ -752,30 +748,9 @@ function makeVector(
values: unknown[],
type?: DataType,
stringAsDictionary?: boolean,
nullable?: boolean,
// biome-ignore lint/suspicious/noExplicitAny: skip
): Vector<any> {
if (type !== undefined) {
// Convert undefined values to null for nullable fields
if (nullable) {
values = values.map((v) => (v === undefined ? null : v));
}
// workaround for: https://github.com/apache/arrow-js/issues/68
if (DataType.isBool(type)) {
const hasNonNullValue = values.some((v) => v !== null && v !== undefined);
if (!hasNonNullValue) {
const nullBitmap = new Uint8Array(Math.ceil(values.length / 8));
const data = makeData({
type: type,
length: values.length,
nullCount: values.length,
nullBitmap,
});
return arrowMakeVector(data);
}
}
// No need for inference, let Arrow create it
if (type instanceof Int) {
if (DataType.isInt(type) && type.bitWidth === 64) {
@@ -900,12 +875,7 @@ async function applyEmbeddingsFromMetadata(
for (const field of schema.fields) {
if (!(field.name in columns)) {
const nullValues = new Array(table.numRows).fill(null);
columns[field.name] = makeVector(
nullValues,
field.type,
undefined,
field.nullable,
);
columns[field.name] = makeVector(nullValues, field.type);
}
}
@@ -969,12 +939,7 @@ async function applyEmbeddings<T>(
} else if (schema != null) {
const destField = schema.fields.find((f) => f.name === destColumn);
if (destField != null) {
newColumns[destColumn] = makeVector(
[],
destField.type,
undefined,
destField.nullable,
);
newColumns[destColumn] = makeVector([], destField.type);
} else {
throw new Error(
`Attempt to apply embeddings to an empty table failed because schema was missing embedding column '${destColumn}'`,

View File

@@ -159,33 +159,17 @@ export abstract class Connection {
*
* Tables will be returned in lexicographical order.
* @param {Partial<TableNamesOptions>} options - options to control the
* paging / start point (backwards compatibility)
*
*/
abstract tableNames(options?: Partial<TableNamesOptions>): Promise<string[]>;
/**
* List all the table names in this database.
*
* Tables will be returned in lexicographical order.
* @param {string[]} namespace - The namespace to list tables from (defaults to root namespace)
* @param {Partial<TableNamesOptions>} options - options to control the
* paging / start point
*
*/
abstract tableNames(
namespace?: string[],
options?: Partial<TableNamesOptions>,
): Promise<string[]>;
abstract tableNames(options?: Partial<TableNamesOptions>): Promise<string[]>;
/**
* Open a table in the database.
* @param {string} name - The name of the table
* @param {string[]} namespace - The namespace of the table (defaults to root namespace)
* @param {Partial<OpenTableOptions>} options - Additional options
*/
abstract openTable(
name: string,
namespace?: string[],
options?: Partial<OpenTableOptions>,
): Promise<Table>;
@@ -194,7 +178,6 @@ export abstract class Connection {
* @param {object} options - The options object.
* @param {string} options.name - The name of the table.
* @param {Data} options.data - Non-empty Array of Records to be inserted into the table
* @param {string[]} namespace - The namespace to create the table in (defaults to root namespace)
*
*/
abstract createTable(
@@ -202,99 +185,40 @@ export abstract class Connection {
name: string;
data: Data;
} & Partial<CreateTableOptions>,
namespace?: string[],
): Promise<Table>;
/**
* Creates a new Table and initialize it with new data.
* @param {string} name - The name of the table.
* @param {Record<string, unknown>[] | TableLike} data - Non-empty Array of Records
* to be inserted into the table
* @param {Partial<CreateTableOptions>} options - Additional options (backwards compatibility)
*/
abstract createTable(
name: string,
data: Record<string, unknown>[] | TableLike,
options?: Partial<CreateTableOptions>,
): Promise<Table>;
/**
* Creates a new Table and initialize it with new data.
* @param {string} name - The name of the table.
* @param {Record<string, unknown>[] | TableLike} data - Non-empty Array of Records
* to be inserted into the table
* @param {string[]} namespace - The namespace to create the table in (defaults to root namespace)
* @param {Partial<CreateTableOptions>} options - Additional options
*/
abstract createTable(
name: string,
data: Record<string, unknown>[] | TableLike,
namespace?: string[],
options?: Partial<CreateTableOptions>,
): Promise<Table>;
/**
* Creates a new empty Table
* @param {string} name - The name of the table.
* @param {Schema} schema - The schema of the table
* @param {Partial<CreateTableOptions>} options - Additional options (backwards compatibility)
*/
abstract createEmptyTable(
name: string,
schema: import("./arrow").SchemaLike,
options?: Partial<CreateTableOptions>,
): Promise<Table>;
/**
* Creates a new empty Table
* @param {string} name - The name of the table.
* @param {Schema} schema - The schema of the table
* @param {string[]} namespace - The namespace to create the table in (defaults to root namespace)
* @param {Partial<CreateTableOptions>} options - Additional options
*/
abstract createEmptyTable(
name: string,
schema: import("./arrow").SchemaLike,
namespace?: string[],
options?: Partial<CreateTableOptions>,
): Promise<Table>;
/**
* Drop an existing table.
* @param {string} name The name of the table to drop.
* @param {string[]} namespace The namespace of the table (defaults to root namespace).
*/
abstract dropTable(name: string, namespace?: string[]): Promise<void>;
abstract dropTable(name: string): Promise<void>;
/**
* Drop all tables in the database.
* @param {string[]} namespace The namespace to drop tables from (defaults to root namespace).
*/
abstract dropAllTables(namespace?: string[]): Promise<void>;
/**
* Clone a table from a source table.
*
* A shallow clone creates a new table that shares the underlying data files
* with the source table but has its own independent manifest. This allows
* both the source and cloned tables to evolve independently while initially
* sharing the same data, deletion, and index files.
*
* @param {string} targetTableName - The name of the target table to create.
* @param {string} sourceUri - The URI of the source table to clone from.
* @param {object} options - Clone options.
* @param {string[]} options.targetNamespace - The namespace for the target table (defaults to root namespace).
* @param {number} options.sourceVersion - The version of the source table to clone.
* @param {string} options.sourceTag - The tag of the source table to clone.
* @param {boolean} options.isShallow - Whether to perform a shallow clone (defaults to true).
*/
abstract cloneTable(
targetTableName: string,
sourceUri: string,
options?: {
targetNamespace?: string[];
sourceVersion?: number;
sourceTag?: string;
isShallow?: boolean;
},
): Promise<Table>;
abstract dropAllTables(): Promise<void>;
}
/** @hideconstructor */
@@ -319,39 +243,16 @@ export class LocalConnection extends Connection {
return this.inner.display();
}
async tableNames(
namespaceOrOptions?: string[] | Partial<TableNamesOptions>,
options?: Partial<TableNamesOptions>,
): Promise<string[]> {
// Detect if first argument is namespace array or options object
let namespace: string[] | undefined;
let tableNamesOptions: Partial<TableNamesOptions> | undefined;
if (Array.isArray(namespaceOrOptions)) {
// First argument is namespace array
namespace = namespaceOrOptions;
tableNamesOptions = options;
} else {
// First argument is options object (backwards compatibility)
namespace = undefined;
tableNamesOptions = namespaceOrOptions;
}
return this.inner.tableNames(
namespace ?? [],
tableNamesOptions?.startAfter,
tableNamesOptions?.limit,
);
async tableNames(options?: Partial<TableNamesOptions>): Promise<string[]> {
return this.inner.tableNames(options?.startAfter, options?.limit);
}
async openTable(
name: string,
namespace?: string[],
options?: Partial<OpenTableOptions>,
): Promise<Table> {
const innerTable = await this.inner.openTable(
name,
namespace ?? [],
cleanseStorageOptions(options?.storageOptions),
options?.indexCacheSize,
);
@@ -359,28 +260,6 @@ export class LocalConnection extends Connection {
return new LocalTable(innerTable);
}
async cloneTable(
targetTableName: string,
sourceUri: string,
options?: {
targetNamespace?: string[];
sourceVersion?: number;
sourceTag?: string;
isShallow?: boolean;
},
): Promise<Table> {
const innerTable = await this.inner.cloneTable(
targetTableName,
sourceUri,
options?.targetNamespace ?? [],
options?.sourceVersion ?? null,
options?.sourceTag ?? null,
options?.isShallow ?? true,
);
return new LocalTable(innerTable);
}
private getStorageOptions(
options?: Partial<CreateTableOptions>,
): Record<string, string> | undefined {
@@ -407,44 +286,14 @@ export class LocalConnection extends Connection {
nameOrOptions:
| string
| ({ name: string; data: Data } & Partial<CreateTableOptions>),
dataOrNamespace?: Record<string, unknown>[] | TableLike | string[],
namespaceOrOptions?: string[] | Partial<CreateTableOptions>,
data?: Record<string, unknown>[] | TableLike,
options?: Partial<CreateTableOptions>,
): Promise<Table> {
if (typeof nameOrOptions !== "string" && "name" in nameOrOptions) {
// First overload: createTable(options, namespace?)
const { name, data, ...createOptions } = nameOrOptions;
const namespace = dataOrNamespace as string[] | undefined;
return this._createTableImpl(name, data, namespace, createOptions);
const { name, data, ...options } = nameOrOptions;
return this.createTable(name, data, options);
}
// Second overload: createTable(name, data, namespace?, options?)
const name = nameOrOptions;
const data = dataOrNamespace as Record<string, unknown>[] | TableLike;
// Detect if third argument is namespace array or options object
let namespace: string[] | undefined;
let createOptions: Partial<CreateTableOptions> | undefined;
if (Array.isArray(namespaceOrOptions)) {
// Third argument is namespace array
namespace = namespaceOrOptions;
createOptions = options;
} else {
// Third argument is options object (backwards compatibility)
namespace = undefined;
createOptions = namespaceOrOptions;
}
return this._createTableImpl(name, data, namespace, createOptions);
}
private async _createTableImpl(
name: string,
data: Data,
namespace?: string[],
options?: Partial<CreateTableOptions>,
): Promise<Table> {
if (data === undefined) {
throw new Error("data is required");
}
@@ -453,10 +302,9 @@ export class LocalConnection extends Connection {
const storageOptions = this.getStorageOptions(options);
const innerTable = await this.inner.createTable(
name,
nameOrOptions,
buf,
mode,
namespace ?? [],
storageOptions,
);
@@ -466,55 +314,39 @@ export class LocalConnection extends Connection {
async createEmptyTable(
name: string,
schema: import("./arrow").SchemaLike,
namespaceOrOptions?: string[] | Partial<CreateTableOptions>,
options?: Partial<CreateTableOptions>,
): Promise<Table> {
// Detect if third argument is namespace array or options object
let namespace: string[] | undefined;
let createOptions: Partial<CreateTableOptions> | undefined;
if (Array.isArray(namespaceOrOptions)) {
// Third argument is namespace array
namespace = namespaceOrOptions;
createOptions = options;
} else {
// Third argument is options object (backwards compatibility)
namespace = undefined;
createOptions = namespaceOrOptions;
}
let mode: string = createOptions?.mode ?? "create";
const existOk = createOptions?.existOk ?? false;
let mode: string = options?.mode ?? "create";
const existOk = options?.existOk ?? false;
if (mode === "create" && existOk) {
mode = "exist_ok";
}
let metadata: Map<string, string> | undefined = undefined;
if (createOptions?.embeddingFunction !== undefined) {
const embeddingFunction = createOptions.embeddingFunction;
if (options?.embeddingFunction !== undefined) {
const embeddingFunction = options.embeddingFunction;
const registry = getRegistry();
metadata = registry.getTableMetadata([embeddingFunction]);
}
const storageOptions = this.getStorageOptions(createOptions);
const storageOptions = this.getStorageOptions(options);
const table = makeEmptyTable(schema, metadata);
const buf = await fromTableToBuffer(table);
const innerTable = await this.inner.createEmptyTable(
name,
buf,
mode,
namespace ?? [],
storageOptions,
);
return new LocalTable(innerTable);
}
async dropTable(name: string, namespace?: string[]): Promise<void> {
return this.inner.dropTable(name, namespace ?? []);
async dropTable(name: string): Promise<void> {
return this.inner.dropTable(name);
}
async dropAllTables(namespace?: string[]): Promise<void> {
return this.inner.dropAllTables(namespace ?? []);
async dropAllTables(): Promise<void> {
return this.inner.dropAllTables();
}
}

View File

@@ -1,253 +0,0 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
/**
* Header providers for LanceDB remote connections.
*
* This module provides a flexible header management framework for LanceDB remote
* connections, allowing users to implement custom header strategies for
* authentication, request tracking, custom metadata, or any other header-based
* requirements.
*
* @module header
*/
/**
* Abstract base class for providing custom headers for each request.
*
* Users can implement this interface to provide dynamic headers for various purposes
* such as authentication (OAuth tokens, API keys), request tracking (correlation IDs),
* custom metadata, or any other header-based requirements. The provider is called
* before each request to ensure fresh header values are always used.
*
* @example
* Simple JWT token provider:
* ```typescript
* class JWTProvider extends HeaderProvider {
* constructor(private token: string) {
* super();
* }
*
* getHeaders(): Record<string, string> {
* return { authorization: `Bearer ${this.token}` };
* }
* }
* ```
*
* @example
* Provider with request tracking:
* ```typescript
* class RequestTrackingProvider extends HeaderProvider {
* constructor(private sessionId: string) {
* super();
* }
*
* getHeaders(): Record<string, string> {
* return {
* "X-Session-Id": this.sessionId,
* "X-Request-Id": `req-${Date.now()}`
* };
* }
* }
* ```
*/
export abstract class HeaderProvider {
/**
* Get the latest headers to be added to requests.
*
* This method is called before each request to the remote LanceDB server.
* Implementations should return headers that will be merged with existing headers.
*
* @returns Dictionary of header names to values to add to the request.
* @throws If unable to fetch headers, the exception will be propagated and the request will fail.
*/
abstract getHeaders(): Record<string, string>;
}
/**
* Example implementation: A simple header provider that returns static headers.
*
* This is an example implementation showing how to create a HeaderProvider
* for cases where headers don't change during the session.
*
* @example
* ```typescript
* const provider = new StaticHeaderProvider({
* authorization: "Bearer my-token",
* "X-Custom-Header": "custom-value"
* });
* const headers = provider.getHeaders();
* // Returns: {authorization: 'Bearer my-token', 'X-Custom-Header': 'custom-value'}
* ```
*/
export class StaticHeaderProvider extends HeaderProvider {
private _headers: Record<string, string>;
/**
* Initialize with static headers.
* @param headers - Headers to return for every request.
*/
constructor(headers: Record<string, string>) {
super();
this._headers = { ...headers };
}
/**
* Return the static headers.
* @returns Copy of the static headers.
*/
getHeaders(): Record<string, string> {
return { ...this._headers };
}
}
/**
* Token response from OAuth provider.
* @public
*/
export interface TokenResponse {
accessToken: string;
expiresIn?: number;
}
/**
* Example implementation: OAuth token provider with automatic refresh.
*
* This is an example implementation showing how to manage OAuth tokens
* with automatic refresh when they expire.
*
* @example
* ```typescript
* async function fetchToken(): Promise<TokenResponse> {
* const response = await fetch("https://oauth.example.com/token", {
* method: "POST",
* body: JSON.stringify({
* grant_type: "client_credentials",
* client_id: "your-client-id",
* client_secret: "your-client-secret"
* }),
* headers: { "Content-Type": "application/json" }
* });
* const data = await response.json();
* return {
* accessToken: data.access_token,
* expiresIn: data.expires_in
* };
* }
*
* const provider = new OAuthHeaderProvider(fetchToken);
* const headers = provider.getHeaders();
* // Returns: {"authorization": "Bearer <your-token>"}
* ```
*/
export class OAuthHeaderProvider extends HeaderProvider {
private _tokenFetcher: () => Promise<TokenResponse> | TokenResponse;
private _refreshBufferSeconds: number;
private _currentToken: string | null = null;
private _tokenExpiresAt: number | null = null;
private _refreshPromise: Promise<void> | null = null;
/**
* Initialize the OAuth provider.
* @param tokenFetcher - Function to fetch new tokens. Should return object with 'accessToken' and optionally 'expiresIn'.
* @param refreshBufferSeconds - Seconds before expiry to refresh token. Default 300 (5 minutes).
*/
constructor(
tokenFetcher: () => Promise<TokenResponse> | TokenResponse,
refreshBufferSeconds: number = 300,
) {
super();
this._tokenFetcher = tokenFetcher;
this._refreshBufferSeconds = refreshBufferSeconds;
}
/**
* Check if token needs refresh.
*/
private _needsRefresh(): boolean {
if (this._currentToken === null) {
return true;
}
if (this._tokenExpiresAt === null) {
// No expiration info, assume token is valid
return false;
}
// Refresh if we're within the buffer time of expiration
const now = Date.now() / 1000;
return now >= this._tokenExpiresAt - this._refreshBufferSeconds;
}
/**
* Refresh the token if it's expired or close to expiring.
*/
private async _refreshTokenIfNeeded(): Promise<void> {
if (!this._needsRefresh()) {
return;
}
// If refresh is already in progress, wait for it
if (this._refreshPromise) {
await this._refreshPromise;
return;
}
// Start refresh
this._refreshPromise = (async () => {
try {
const tokenData = await this._tokenFetcher();
this._currentToken = tokenData.accessToken;
if (!this._currentToken) {
throw new Error("Token fetcher did not return 'accessToken'");
}
// Set expiration if provided
if (tokenData.expiresIn) {
this._tokenExpiresAt = Date.now() / 1000 + tokenData.expiresIn;
} else {
// Token doesn't expire or expiration unknown
this._tokenExpiresAt = null;
}
} finally {
this._refreshPromise = null;
}
})();
await this._refreshPromise;
}
/**
* Get OAuth headers, refreshing token if needed.
* Note: This is synchronous for now as the Rust implementation expects sync.
* In a real implementation, this would need to handle async properly.
* @returns Headers with Bearer token authorization.
* @throws If unable to fetch or refresh token.
*/
getHeaders(): Record<string, string> {
// For simplicity in this example, we assume the token is already fetched
// In a real implementation, this would need to handle the async nature properly
if (!this._currentToken && !this._refreshPromise) {
// Synchronously trigger refresh - this is a limitation of the current implementation
throw new Error(
"Token not initialized. Call refreshToken() first or use async initialization.",
);
}
if (!this._currentToken) {
throw new Error("Failed to obtain OAuth token");
}
return { authorization: `Bearer ${this._currentToken}` };
}
/**
* Manually refresh the token.
* Call this before using getHeaders() to ensure token is available.
*/
async refreshToken(): Promise<void> {
this._currentToken = null; // Force refresh
await this._refreshTokenIfNeeded();
}
}

View File

@@ -10,15 +10,9 @@ import {
import {
ConnectionOptions,
Connection as LanceDbConnection,
JsHeaderProvider as NativeJsHeaderProvider,
Session,
} from "./native.js";
import { HeaderProvider } from "./header";
// Re-export native header provider for use with connectWithHeaderProvider
export { JsHeaderProvider as NativeJsHeaderProvider } from "./native.js";
export {
AddColumnsSql,
ConnectionOptions,
@@ -27,7 +21,6 @@ export {
ClientConfig,
TimeoutConfig,
RetryConfig,
TlsConfig,
OptimizeStats,
CompactionStats,
RemovalStats,
@@ -66,7 +59,6 @@ export {
Query,
QueryBase,
VectorQuery,
TakeQuery,
QueryExecutionOptions,
FullTextSearchOptions,
RecordBatchIterator,
@@ -100,13 +92,6 @@ export {
ColumnAlteration,
} from "./table";
export {
HeaderProvider,
StaticHeaderProvider,
OAuthHeaderProvider,
TokenResponse,
} from "./header";
export { MergeInsertBuilder, WriteExecutionOptions } from "./merge";
export * as embedding from "./embedding";
@@ -145,27 +130,11 @@ export { IntoSql, packBits } from "./util";
* {storageOptions: {timeout: "60s"}
* });
* ```
* @example
* Using with a header provider for per-request authentication:
* ```ts
* const provider = new StaticHeaderProvider({
* "X-API-Key": "my-key"
* });
* const conn = await connectWithHeaderProvider(
* "db://host:port",
* options,
* provider
* );
* ```
*/
export async function connect(
uri: string,
options?: Partial<ConnectionOptions>,
session?: Session,
headerProvider?:
| HeaderProvider
| (() => Record<string, string>)
| (() => Promise<Record<string, string>>),
): Promise<Connection>;
/**
* Connect to a LanceDB instance at the given URI.
@@ -199,58 +168,18 @@ export async function connect(
): Promise<Connection>;
export async function connect(
uriOrOptions: string | (Partial<ConnectionOptions> & { uri: string }),
optionsOrSession?: Partial<ConnectionOptions> | Session,
sessionOrHeaderProvider?:
| Session
| HeaderProvider
| (() => Record<string, string>)
| (() => Promise<Record<string, string>>),
headerProvider?:
| HeaderProvider
| (() => Record<string, string>)
| (() => Promise<Record<string, string>>),
options?: Partial<ConnectionOptions>,
): Promise<Connection> {
let uri: string | undefined;
let finalOptions: Partial<ConnectionOptions> = {};
let finalHeaderProvider:
| HeaderProvider
| (() => Record<string, string>)
| (() => Promise<Record<string, string>>)
| undefined;
if (typeof uriOrOptions !== "string") {
// First overload: connect(options)
const { uri: uri_, ...opts } = uriOrOptions;
uri = uri_;
finalOptions = opts;
} else {
// Second overload: connect(uri, options?, session?, headerProvider?)
uri = uriOrOptions;
// Handle optionsOrSession parameter
if (optionsOrSession && "inner" in optionsOrSession) {
// Second param is session, so no options provided
finalOptions = {};
} else {
// Second param is options
finalOptions = (optionsOrSession as Partial<ConnectionOptions>) || {};
}
// Handle sessionOrHeaderProvider parameter
if (
sessionOrHeaderProvider &&
(typeof sessionOrHeaderProvider === "function" ||
"getHeaders" in sessionOrHeaderProvider)
) {
// Third param is header provider
finalHeaderProvider = sessionOrHeaderProvider as
| HeaderProvider
| (() => Record<string, string>)
| (() => Promise<Record<string, string>>);
} else {
// Third param is session, header provider is fourth param
finalHeaderProvider = headerProvider;
}
finalOptions = options || {};
}
if (!uri) {
@@ -261,26 +190,6 @@ export async function connect(
(<ConnectionOptions>finalOptions).storageOptions = cleanseStorageOptions(
(<ConnectionOptions>finalOptions).storageOptions,
);
// Create native header provider if one was provided
let nativeProvider: NativeJsHeaderProvider | undefined;
if (finalHeaderProvider) {
if (typeof finalHeaderProvider === "function") {
nativeProvider = new NativeJsHeaderProvider(finalHeaderProvider);
} else if (
finalHeaderProvider &&
typeof finalHeaderProvider.getHeaders === "function"
) {
nativeProvider = new NativeJsHeaderProvider(async () =>
finalHeaderProvider.getHeaders(),
);
}
}
const nativeConn = await LanceDbConnection.new(
uri,
finalOptions,
nativeProvider,
);
const nativeConn = await LanceDbConnection.new(uri, finalOptions);
return new LocalConnection(nativeConn);
}

View File

@@ -700,27 +700,5 @@ export interface IndexOptions {
*/
replace?: boolean;
/**
* Timeout in seconds to wait for index creation to complete.
*
* If not specified, the method will return immediately after starting the index creation.
*/
waitTimeoutSeconds?: number;
/**
* Optional custom name for the index.
*
* If not provided, a default name will be generated based on the column name.
*/
name?: string;
/**
* Whether to train the index with existing data.
*
* If true (default), the index will be trained with existing data in the table.
* If false, the index will be created empty and populated as new data is added.
*
* Note: This option is only supported for scalar indices. Vector indices always train.
*/
train?: boolean;
}

View File

@@ -15,7 +15,6 @@ import {
RecordBatchIterator as NativeBatchIterator,
Query as NativeQuery,
Table as NativeTable,
TakeQuery as NativeTakeQuery,
VectorQuery as NativeVectorQuery,
} from "./native";
import { Reranker } from "./rerankers";
@@ -51,7 +50,7 @@ export class RecordBatchIterator implements AsyncIterator<RecordBatch> {
/* eslint-enable */
class RecordBatchIterable<
NativeQueryType extends NativeQuery | NativeVectorQuery | NativeTakeQuery,
NativeQueryType extends NativeQuery | NativeVectorQuery,
> implements AsyncIterable<RecordBatch>
{
private inner: NativeQueryType;
@@ -108,9 +107,8 @@ export interface FullTextSearchOptions {
*
* @hideconstructor
*/
export class QueryBase<
NativeQueryType extends NativeQuery | NativeVectorQuery | NativeTakeQuery,
> implements AsyncIterable<RecordBatch>
export class QueryBase<NativeQueryType extends NativeQuery | NativeVectorQuery>
implements AsyncIterable<RecordBatch>
{
/**
* @hidden
@@ -135,6 +133,56 @@ export class QueryBase<
fn(this.inner);
}
}
/**
* A filter statement to be applied to this query.
*
* The filter should be supplied as an SQL query string. For example:
* @example
* x > 10
* y > 0 AND y < 100
* x > 5 OR y = 'test'
*
* Filtering performance can often be improved by creating a scalar index
* on the filter column(s).
*/
where(predicate: string): this {
this.doCall((inner: NativeQueryType) => inner.onlyIf(predicate));
return this;
}
/**
* A filter statement to be applied to this query.
* @see where
* @deprecated Use `where` instead
*/
filter(predicate: string): this {
return this.where(predicate);
}
fullTextSearch(
query: string | FullTextQuery,
options?: Partial<FullTextSearchOptions>,
): this {
let columns: string[] | null = null;
if (options) {
if (typeof options.columns === "string") {
columns = [options.columns];
} else if (Array.isArray(options.columns)) {
columns = options.columns;
}
}
this.doCall((inner: NativeQueryType) => {
if (typeof query === "string") {
inner.fullTextSearch({
query: query,
columns: columns,
});
} else {
inner.fullTextSearch({ query: query.inner });
}
});
return this;
}
/**
* Return only the specified columns.
@@ -193,6 +241,33 @@ export class QueryBase<
return this;
}
/**
* Set the maximum number of results to return.
*
* By default, a plain search has no limit. If this method is not
* called then every valid row from the table will be returned.
*/
limit(limit: number): this {
this.doCall((inner: NativeQueryType) => inner.limit(limit));
return this;
}
offset(offset: number): this {
this.doCall((inner: NativeQueryType) => inner.offset(offset));
return this;
}
/**
* Skip searching un-indexed data. This can make search faster, but will miss
* any data that is not yet indexed.
*
* Use {@link Table#optimize} to index all un-indexed data.
*/
fastSearch(): this {
this.doCall((inner: NativeQueryType) => inner.fastSearch());
return this;
}
/**
* Whether to return the row id in the results.
*
@@ -328,100 +403,6 @@ export class QueryBase<
}
}
export class StandardQueryBase<
NativeQueryType extends NativeQuery | NativeVectorQuery,
>
extends QueryBase<NativeQueryType>
implements ExecutableQuery
{
constructor(inner: NativeQueryType | Promise<NativeQueryType>) {
super(inner);
}
/**
* A filter statement to be applied to this query.
*
* The filter should be supplied as an SQL query string. For example:
* @example
* x > 10
* y > 0 AND y < 100
* x > 5 OR y = 'test'
*
* Filtering performance can often be improved by creating a scalar index
* on the filter column(s).
*/
where(predicate: string): this {
this.doCall((inner: NativeQueryType) => inner.onlyIf(predicate));
return this;
}
/**
* A filter statement to be applied to this query.
* @see where
* @deprecated Use `where` instead
*/
filter(predicate: string): this {
return this.where(predicate);
}
fullTextSearch(
query: string | FullTextQuery,
options?: Partial<FullTextSearchOptions>,
): this {
let columns: string[] | null = null;
if (options) {
if (typeof options.columns === "string") {
columns = [options.columns];
} else if (Array.isArray(options.columns)) {
columns = options.columns;
}
}
this.doCall((inner: NativeQueryType) => {
if (typeof query === "string") {
inner.fullTextSearch({
query: query,
columns: columns,
});
} else {
inner.fullTextSearch({ query: query.inner });
}
});
return this;
}
/**
* Set the maximum number of results to return.
*
* By default, a plain search has no limit. If this method is not
* called then every valid row from the table will be returned.
*/
limit(limit: number): this {
this.doCall((inner: NativeQueryType) => inner.limit(limit));
return this;
}
/**
* Set the number of rows to skip before returning results.
*
* This is useful for pagination.
*/
offset(offset: number): this {
this.doCall((inner: NativeQueryType) => inner.offset(offset));
return this;
}
/**
* Skip searching un-indexed data. This can make search faster, but will miss
* any data that is not yet indexed.
*
* Use {@link Table#optimize} to index all un-indexed data.
*/
fastSearch(): this {
this.doCall((inner: NativeQueryType) => inner.fastSearch());
return this;
}
}
/**
* An interface for a query that can be executed
*
@@ -438,7 +419,7 @@ export interface ExecutableQuery {}
*
* @hideconstructor
*/
export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
export class VectorQuery extends QueryBase<NativeVectorQuery> {
/**
* @hidden
*/
@@ -698,24 +679,13 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
}
}
/**
* A query that returns a subset of the rows in the table.
*
* @hideconstructor
*/
export class TakeQuery extends QueryBase<NativeTakeQuery> {
constructor(inner: NativeTakeQuery) {
super(inner);
}
}
/** A builder for LanceDB queries.
*
* @see {@link Table#query}, {@link Table#search}
*
* @hideconstructor
*/
export class Query extends StandardQueryBase<NativeQuery> {
export class Query extends QueryBase<NativeQuery> {
/**
* @hidden
*/

View File

@@ -35,7 +35,6 @@ import {
import {
FullTextQuery,
Query,
TakeQuery,
VectorQuery,
instanceOfFullTextQuery,
} from "./query";
@@ -337,20 +336,6 @@ export abstract class Table {
*/
abstract query(): Query;
/**
* Create a query that returns a subset of the rows in the table.
* @param offsets The offsets of the rows to return.
* @returns A builder that can be used to parameterize the query.
*/
abstract takeOffsets(offsets: number[]): TakeQuery;
/**
* Create a query that returns a subset of the rows in the table.
* @param rowIds The row ids of the rows to return.
* @returns A builder that can be used to parameterize the query.
*/
abstract takeRowIds(rowIds: number[]): TakeQuery;
/**
* Create a search query to find the nearest neighbors
* of the given query
@@ -662,8 +647,6 @@ export class LocalTable extends Table {
column,
options?.replace,
options?.waitTimeoutSeconds,
options?.name,
options?.train,
);
}
@@ -682,14 +665,6 @@ export class LocalTable extends Table {
await this.inner.waitForIndex(indexNames, timeoutSeconds);
}
takeOffsets(offsets: number[]): TakeQuery {
return new TakeQuery(this.inner.takeOffsets(offsets));
}
takeRowIds(rowIds: number[]): TakeQuery {
return new TakeQuery(this.inner.takeRowIds(rowIds));
}
query(): Query {
return new Query(this.inner);
}

View File

@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-darwin-arm64",
"version": "0.22.1-beta.3",
"version": "0.21.2",
"os": ["darwin"],
"cpu": ["arm64"],
"main": "lancedb.darwin-arm64.node",

View File

@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-darwin-x64",
"version": "0.22.1-beta.3",
"version": "0.21.2",
"os": ["darwin"],
"cpu": ["x64"],
"main": "lancedb.darwin-x64.node",

View File

@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-gnu",
"version": "0.22.1-beta.3",
"version": "0.21.2",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-gnu.node",

View File

@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-musl",
"version": "0.22.1-beta.3",
"version": "0.21.2",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-musl.node",

View File

@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-gnu",
"version": "0.22.1-beta.3",
"version": "0.21.2",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-gnu.node",

View File

@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-musl",
"version": "0.22.1-beta.3",
"version": "0.21.2",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-musl.node",

View File

@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-arm64-msvc",
"version": "0.22.1-beta.3",
"version": "0.21.2",
"os": [
"win32"
],

View File

@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-x64-msvc",
"version": "0.22.1-beta.3",
"version": "0.21.2",
"os": ["win32"],
"cpu": ["x64"],
"main": "lancedb.win32-x64-msvc.node",

228
nodejs/package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "@lancedb/lancedb",
"version": "0.22.1-beta.3",
"version": "0.21.2",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@lancedb/lancedb",
"version": "0.22.1-beta.3",
"version": "0.21.2",
"cpu": [
"x64",
"arm64"
@@ -5549,11 +5549,10 @@
"dev": true
},
"node_modules/brace-expansion": {
"version": "1.1.12",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz",
"integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==",
"version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
"dev": true,
"license": "MIT",
"dependencies": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
@@ -5630,20 +5629,6 @@
"integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==",
"dev": true
},
"node_modules/call-bind-apply-helpers": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz",
"integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==",
"devOptional": true,
"license": "MIT",
"dependencies": {
"es-errors": "^1.3.0",
"function-bind": "^1.1.2"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/camelcase": {
"version": "5.3.1",
"resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz",
@@ -6047,21 +6032,6 @@
"node": ">=6.0.0"
}
},
"node_modules/dunder-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
"integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==",
"devOptional": true,
"license": "MIT",
"dependencies": {
"call-bind-apply-helpers": "^1.0.1",
"es-errors": "^1.3.0",
"gopd": "^1.2.0"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/eastasianwidth": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz",
@@ -6101,55 +6071,6 @@
"is-arrayish": "^0.2.1"
}
},
"node_modules/es-define-property": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
"integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==",
"devOptional": true,
"license": "MIT",
"engines": {
"node": ">= 0.4"
}
},
"node_modules/es-errors": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz",
"integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==",
"devOptional": true,
"license": "MIT",
"engines": {
"node": ">= 0.4"
}
},
"node_modules/es-object-atoms": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz",
"integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==",
"devOptional": true,
"license": "MIT",
"dependencies": {
"es-errors": "^1.3.0"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/es-set-tostringtag": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz",
"integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==",
"devOptional": true,
"license": "MIT",
"dependencies": {
"es-errors": "^1.3.0",
"get-intrinsic": "^1.2.6",
"has-tostringtag": "^1.0.2",
"hasown": "^2.0.2"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/escalade": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz",
@@ -6589,16 +6510,13 @@
}
},
"node_modules/form-data": {
"version": "4.0.4",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz",
"integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==",
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
"integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
"devOptional": true,
"license": "MIT",
"dependencies": {
"asynckit": "^0.4.0",
"combined-stream": "^1.0.8",
"es-set-tostringtag": "^2.1.0",
"hasown": "^2.0.2",
"mime-types": "^2.1.12"
},
"engines": {
@@ -6657,7 +6575,7 @@
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
"integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==",
"devOptional": true,
"dev": true,
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
@@ -6680,31 +6598,6 @@
"node": "6.* || 8.* || >= 10.*"
}
},
"node_modules/get-intrinsic": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz",
"integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==",
"devOptional": true,
"license": "MIT",
"dependencies": {
"call-bind-apply-helpers": "^1.0.2",
"es-define-property": "^1.0.1",
"es-errors": "^1.3.0",
"es-object-atoms": "^1.1.1",
"function-bind": "^1.1.2",
"get-proto": "^1.0.1",
"gopd": "^1.2.0",
"has-symbols": "^1.1.0",
"hasown": "^2.0.2",
"math-intrinsics": "^1.1.0"
},
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/get-package-type": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz",
@@ -6714,20 +6607,6 @@
"node": ">=8.0.0"
}
},
"node_modules/get-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz",
"integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==",
"devOptional": true,
"license": "MIT",
"dependencies": {
"dunder-proto": "^1.0.1",
"es-object-atoms": "^1.0.0"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/get-stream": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz",
@@ -6819,19 +6698,6 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/gopd": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
"integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==",
"devOptional": true,
"license": "MIT",
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/graceful-fs": {
"version": "4.2.11",
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz",
@@ -6858,41 +6724,11 @@
"node": ">=8"
}
},
"node_modules/has-symbols": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz",
"integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==",
"devOptional": true,
"license": "MIT",
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/has-tostringtag": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz",
"integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==",
"devOptional": true,
"license": "MIT",
"dependencies": {
"has-symbols": "^1.0.3"
},
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/hasown": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz",
"integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==",
"devOptional": true,
"license": "MIT",
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.0.tgz",
"integrity": "sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==",
"dev": true,
"dependencies": {
"function-bind": "^1.1.2"
},
@@ -8107,16 +7943,6 @@
"integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
"dev": true
},
"node_modules/math-intrinsics": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",
"integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==",
"devOptional": true,
"license": "MIT",
"engines": {
"node": ">= 0.4"
}
},
"node_modules/md5": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz",
@@ -8227,10 +8053,9 @@
}
},
"node_modules/minizlib/node_modules/brace-expansion": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz",
"integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==",
"license": "MIT",
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
"integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
"optional": true,
"dependencies": {
"balanced-match": "^1.0.0"
@@ -9376,11 +9201,10 @@
"dev": true
},
"node_modules/tmp": {
"version": "0.2.5",
"resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.5.tgz",
"integrity": "sha512-voyz6MApa1rQGUxT3E+BK7/ROe8itEx7vD8/HEvt4xwXucvQ5G5oeEiHkmHZJuBO21RpOf+YYm9MOivj709jow==",
"version": "0.2.3",
"resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.3.tgz",
"integrity": "sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=14.14"
}
@@ -9525,11 +9349,10 @@
}
},
"node_modules/typedoc/node_modules/brace-expansion": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz",
"integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==",
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
"integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
"dev": true,
"license": "MIT",
"dependencies": {
"balanced-match": "^1.0.0"
}
@@ -9779,11 +9602,10 @@
}
},
"node_modules/typescript-eslint/node_modules/brace-expansion": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz",
"integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==",
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
"integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
"dev": true,
"license": "MIT",
"dependencies": {
"balanced-match": "^1.0.0"
}

View File

@@ -11,7 +11,7 @@
"ann"
],
"private": false,
"version": "0.22.1-beta.3",
"version": "0.21.2",
"main": "dist/index.js",
"exports": {
".": "./dist/index.js",

View File

@@ -2,14 +2,12 @@
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::collections::HashMap;
use std::sync::Arc;
use lancedb::database::CreateTableMode;
use napi::bindgen_prelude::*;
use napi_derive::*;
use crate::error::NapiErrorExt;
use crate::header::JsHeaderProvider;
use crate::table::Table;
use crate::ConnectionOptions;
use lancedb::connection::{ConnectBuilder, Connection as LanceDBConnection};
@@ -47,11 +45,7 @@ impl Connection {
impl Connection {
/// Create a new Connection instance from the given URI.
#[napi(factory)]
pub async fn new(
uri: String,
options: ConnectionOptions,
header_provider: Option<&JsHeaderProvider>,
) -> napi::Result<Self> {
pub async fn new(uri: String, options: ConnectionOptions) -> napi::Result<Self> {
let mut builder = ConnectBuilder::new(&uri);
if let Some(interval) = options.read_consistency_interval {
builder =
@@ -63,16 +57,8 @@ impl Connection {
}
}
// Create client config, optionally with header provider
let client_config = options.client_config.unwrap_or_default();
let mut rust_config: lancedb::remote::ClientConfig = client_config.into();
if let Some(provider) = header_provider {
rust_config.header_provider =
Some(Arc::new(provider.clone()) as Arc<dyn lancedb::remote::HeaderProvider>);
}
builder = builder.client_config(rust_config);
builder = builder.client_config(client_config.into());
if let Some(api_key) = options.api_key {
builder = builder.api_key(&api_key);
@@ -114,12 +100,10 @@ impl Connection {
#[napi(catch_unwind)]
pub async fn table_names(
&self,
namespace: Vec<String>,
start_after: Option<String>,
limit: Option<u32>,
) -> napi::Result<Vec<String>> {
let mut op = self.get_inner()?.table_names();
op = op.namespace(namespace);
if let Some(start_after) = start_after {
op = op.start_after(start_after);
}
@@ -141,7 +125,6 @@ impl Connection {
name: String,
buf: Buffer,
mode: String,
namespace: Vec<String>,
storage_options: Option<HashMap<String, String>>,
) -> napi::Result<Table> {
let batches = ipc_file_to_batches(buf.to_vec())
@@ -149,8 +132,6 @@ impl Connection {
let mode = Self::parse_create_mode_str(&mode)?;
let mut builder = self.get_inner()?.create_table(&name, batches).mode(mode);
builder = builder.namespace(namespace);
if let Some(storage_options) = storage_options {
for (key, value) in storage_options {
builder = builder.storage_option(key, value);
@@ -166,7 +147,6 @@ impl Connection {
name: String,
schema_buf: Buffer,
mode: String,
namespace: Vec<String>,
storage_options: Option<HashMap<String, String>>,
) -> napi::Result<Table> {
let schema = ipc_file_to_schema(schema_buf.to_vec()).map_err(|e| {
@@ -177,9 +157,6 @@ impl Connection {
.get_inner()?
.create_empty_table(&name, schema)
.mode(mode);
builder = builder.namespace(namespace);
if let Some(storage_options) = storage_options {
for (key, value) in storage_options {
builder = builder.storage_option(key, value);
@@ -193,14 +170,10 @@ impl Connection {
pub async fn open_table(
&self,
name: String,
namespace: Vec<String>,
storage_options: Option<HashMap<String, String>>,
index_cache_size: Option<u32>,
) -> napi::Result<Table> {
let mut builder = self.get_inner()?.open_table(&name);
builder = builder.namespace(namespace);
if let Some(storage_options) = storage_options {
for (key, value) in storage_options {
builder = builder.storage_option(key, value);
@@ -213,50 +186,14 @@ impl Connection {
Ok(Table::new(tbl))
}
#[napi(catch_unwind)]
pub async fn clone_table(
&self,
target_table_name: String,
source_uri: String,
target_namespace: Vec<String>,
source_version: Option<i64>,
source_tag: Option<String>,
is_shallow: bool,
) -> napi::Result<Table> {
let mut builder = self
.get_inner()?
.clone_table(&target_table_name, &source_uri);
builder = builder.target_namespace(target_namespace);
if let Some(version) = source_version {
builder = builder.source_version(version as u64);
}
if let Some(tag) = source_tag {
builder = builder.source_tag(tag);
}
builder = builder.is_shallow(is_shallow);
let tbl = builder.execute().await.default_error()?;
Ok(Table::new(tbl))
}
/// Drop table with the name. Or raise an error if the table does not exist.
#[napi(catch_unwind)]
pub async fn drop_table(&self, name: String, namespace: Vec<String>) -> napi::Result<()> {
self.get_inner()?
.drop_table(&name, &namespace)
.await
.default_error()
pub async fn drop_table(&self, name: String) -> napi::Result<()> {
self.get_inner()?.drop_table(&name).await.default_error()
}
#[napi(catch_unwind)]
pub async fn drop_all_tables(&self, namespace: Vec<String>) -> napi::Result<()> {
self.get_inner()?
.drop_all_tables(&namespace)
.await
.default_error()
pub async fn drop_all_tables(&self) -> napi::Result<()> {
self.get_inner()?.drop_all_tables().await.default_error()
}
}

View File

@@ -1,71 +0,0 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use napi::{
bindgen_prelude::*,
threadsafe_function::{ErrorStrategy, ThreadsafeFunction},
};
use napi_derive::napi;
use std::collections::HashMap;
use std::sync::Arc;
/// JavaScript HeaderProvider implementation that wraps a JavaScript callback.
/// This is the only native header provider - all header provider implementations
/// should provide a JavaScript function that returns headers.
#[napi]
pub struct JsHeaderProvider {
get_headers_fn: Arc<ThreadsafeFunction<(), ErrorStrategy::CalleeHandled>>,
}
impl Clone for JsHeaderProvider {
fn clone(&self) -> Self {
Self {
get_headers_fn: self.get_headers_fn.clone(),
}
}
}
#[napi]
impl JsHeaderProvider {
/// Create a new JsHeaderProvider from a JavaScript callback
#[napi(constructor)]
pub fn new(get_headers_callback: JsFunction) -> Result<Self> {
let get_headers_fn = get_headers_callback
.create_threadsafe_function(0, |ctx| Ok(vec![ctx.value]))
.map_err(|e| {
Error::new(
Status::GenericFailure,
format!("Failed to create threadsafe function: {}", e),
)
})?;
Ok(Self {
get_headers_fn: Arc::new(get_headers_fn),
})
}
}
#[cfg(feature = "remote")]
#[async_trait::async_trait]
impl lancedb::remote::HeaderProvider for JsHeaderProvider {
async fn get_headers(&self) -> lancedb::error::Result<HashMap<String, String>> {
// Call the JavaScript function asynchronously
let promise: Promise<HashMap<String, String>> =
self.get_headers_fn.call_async(Ok(())).await.map_err(|e| {
lancedb::error::Error::Runtime {
message: format!("Failed to call JavaScript get_headers: {}", e),
}
})?;
// Await the promise result
promise.await.map_err(|e| lancedb::error::Error::Runtime {
message: format!("JavaScript get_headers failed: {}", e),
})
}
}
impl std::fmt::Debug for JsHeaderProvider {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "JsHeaderProvider")
}
}

View File

@@ -8,7 +8,6 @@ use napi_derive::*;
mod connection;
mod error;
mod header;
mod index;
mod iterator;
pub mod merge;

View File

@@ -12,7 +12,6 @@ use lancedb::query::Query as LanceDbQuery;
use lancedb::query::QueryBase;
use lancedb::query::QueryExecutionOptions;
use lancedb::query::Select;
use lancedb::query::TakeQuery as LanceDbTakeQuery;
use lancedb::query::VectorQuery as LanceDbVectorQuery;
use napi::bindgen_prelude::*;
use napi_derive::napi;
@@ -320,79 +319,6 @@ impl VectorQuery {
}
}
#[napi]
pub struct TakeQuery {
inner: LanceDbTakeQuery,
}
#[napi]
impl TakeQuery {
pub fn new(query: LanceDbTakeQuery) -> Self {
Self { inner: query }
}
#[napi]
pub fn select(&mut self, columns: Vec<(String, String)>) {
self.inner = self.inner.clone().select(Select::dynamic(&columns));
}
#[napi]
pub fn select_columns(&mut self, columns: Vec<String>) {
self.inner = self.inner.clone().select(Select::columns(&columns));
}
#[napi]
pub fn with_row_id(&mut self) {
self.inner = self.inner.clone().with_row_id();
}
#[napi(catch_unwind)]
pub async fn execute(
&self,
max_batch_length: Option<u32>,
timeout_ms: Option<u32>,
) -> napi::Result<RecordBatchIterator> {
let mut execution_opts = QueryExecutionOptions::default();
if let Some(max_batch_length) = max_batch_length {
execution_opts.max_batch_length = max_batch_length;
}
if let Some(timeout_ms) = timeout_ms {
execution_opts.timeout = Some(std::time::Duration::from_millis(timeout_ms as u64))
}
let inner_stream = self
.inner
.execute_with_options(execution_opts)
.await
.map_err(|e| {
napi::Error::from_reason(format!(
"Failed to execute query stream: {}",
convert_error(&e)
))
})?;
Ok(RecordBatchIterator::new(inner_stream))
}
#[napi]
pub async fn explain_plan(&self, verbose: bool) -> napi::Result<String> {
self.inner.explain_plan(verbose).await.map_err(|e| {
napi::Error::from_reason(format!(
"Failed to retrieve the query plan: {}",
convert_error(&e)
))
})
}
#[napi(catch_unwind)]
pub async fn analyze_plan(&self) -> napi::Result<String> {
self.inner.analyze_plan().await.map_err(|e| {
napi::Error::from_reason(format!(
"Failed to execute analyze plan: {}",
convert_error(&e)
))
})
}
}
#[napi]
#[derive(Debug, Clone)]
pub struct JsFullTextQuery {
@@ -480,7 +406,6 @@ impl JsFullTextQuery {
}
#[napi(factory)]
#[allow(clippy::use_self)] // NAPI doesn't allow Self here but clippy reports it
pub fn boolean_query(queries: Vec<(String, &JsFullTextQuery)>) -> napi::Result<Self> {
let mut sub_queries = Vec::with_capacity(queries.len());
for (occur, q) in queries {

View File

@@ -69,20 +69,6 @@ pub struct RetryConfig {
pub statuses: Option<Vec<u16>>,
}
/// TLS/mTLS configuration for the remote HTTP client.
#[napi(object)]
#[derive(Debug, Default)]
pub struct TlsConfig {
/// Path to the client certificate file (PEM format) for mTLS authentication.
pub cert_file: Option<String>,
/// Path to the client private key file (PEM format) for mTLS authentication.
pub key_file: Option<String>,
/// Path to the CA certificate file (PEM format) for server verification.
pub ssl_ca_cert: Option<String>,
/// Whether to verify the hostname in the server's certificate.
pub assert_hostname: Option<bool>,
}
#[napi(object)]
#[derive(Debug, Default)]
pub struct ClientConfig {
@@ -90,8 +76,6 @@ pub struct ClientConfig {
pub retry_config: Option<RetryConfig>,
pub timeout_config: Option<TimeoutConfig>,
pub extra_headers: Option<HashMap<String, String>>,
pub id_delimiter: Option<String>,
pub tls_config: Option<TlsConfig>,
}
impl From<TimeoutConfig> for lancedb::remote::TimeoutConfig {
@@ -122,17 +106,6 @@ impl From<RetryConfig> for lancedb::remote::RetryConfig {
}
}
impl From<TlsConfig> for lancedb::remote::TlsConfig {
fn from(config: TlsConfig) -> Self {
Self {
cert_file: config.cert_file,
key_file: config.key_file,
ssl_ca_cert: config.ssl_ca_cert,
assert_hostname: config.assert_hostname.unwrap_or(true),
}
}
}
impl From<ClientConfig> for lancedb::remote::ClientConfig {
fn from(config: ClientConfig) -> Self {
Self {
@@ -142,9 +115,6 @@ impl From<ClientConfig> for lancedb::remote::ClientConfig {
retry_config: config.retry_config.map(Into::into).unwrap_or_default(),
timeout_config: config.timeout_config.map(Into::into).unwrap_or_default(),
extra_headers: config.extra_headers.unwrap_or_default(),
id_delimiter: config.id_delimiter,
tls_config: config.tls_config.map(Into::into),
header_provider: None, // the header provider is set separately later
}
}
}

View File

@@ -94,7 +94,7 @@ impl napi::bindgen_prelude::FromNapiValue for Session {
env: napi::sys::napi_env,
napi_val: napi::sys::napi_value,
) -> napi::Result<Self> {
let object: napi::bindgen_prelude::ClassInstance<Self> =
let object: napi::bindgen_prelude::ClassInstance<Session> =
napi::bindgen_prelude::ClassInstance::from_napi_value(env, napi_val)?;
let copy = object.clone();
Ok(copy)

View File

@@ -15,7 +15,7 @@ use napi_derive::napi;
use crate::error::NapiErrorExt;
use crate::index::Index;
use crate::merge::NativeMergeInsertBuilder;
use crate::query::{Query, TakeQuery, VectorQuery};
use crate::query::{Query, VectorQuery};
#[napi]
pub struct Table {
@@ -114,8 +114,6 @@ impl Table {
column: String,
replace: Option<bool>,
wait_timeout_s: Option<i64>,
name: Option<String>,
train: Option<bool>,
) -> napi::Result<()> {
let lancedb_index = if let Some(index) = index {
index.consume()?
@@ -130,12 +128,6 @@ impl Table {
builder =
builder.wait_timeout(std::time::Duration::from_secs(timeout.try_into().unwrap()));
}
if let Some(name) = name {
builder = builder.name(name);
}
if let Some(train) = train {
builder = builder.train(train);
}
builder.execute().await.default_error()
}
@@ -195,44 +187,6 @@ impl Table {
Ok(Query::new(self.inner_ref()?.query()))
}
#[napi(catch_unwind)]
pub fn take_offsets(&self, offsets: Vec<i64>) -> napi::Result<TakeQuery> {
Ok(TakeQuery::new(
self.inner_ref()?.take_offsets(
offsets
.into_iter()
.map(|o| {
u64::try_from(o).map_err(|e| {
napi::Error::from_reason(format!(
"Failed to convert offset to u64: {}",
e
))
})
})
.collect::<Result<Vec<_>>>()?,
),
))
}
#[napi(catch_unwind)]
pub fn take_row_ids(&self, row_ids: Vec<i64>) -> napi::Result<TakeQuery> {
Ok(TakeQuery::new(
self.inner_ref()?.take_row_ids(
row_ids
.into_iter()
.map(|o| {
u64::try_from(o).map_err(|e| {
napi::Error::from_reason(format!(
"Failed to convert row id to u64: {}",
e
))
})
})
.collect::<Result<Vec<_>>>()?,
),
))
}
#[napi(catch_unwind)]
pub fn vector_search(&self, vector: Float32Array) -> napi::Result<VectorQuery> {
self.query()?.nearest_to(vector)

View File

@@ -2,7 +2,6 @@
"intentionallyNotExported": [
"lancedb/native.d.ts:Query",
"lancedb/native.d.ts:VectorQuery",
"lancedb/native.d.ts:TakeQuery",
"lancedb/native.d.ts:RecordBatchIterator",
"lancedb/native.d.ts:NativeMergeInsertBuilder"
],

View File

@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.25.1"
current_version = "0.24.2"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.

View File

@@ -1,6 +1,6 @@
[package]
name = "lancedb-python"
version = "0.25.1"
version = "0.24.2"
edition.workspace = true
description = "Python bindings for LanceDB"
license.workspace = true
@@ -15,7 +15,6 @@ crate-type = ["cdylib"]
[dependencies]
arrow = { version = "55.1", features = ["pyarrow"] }
async-trait = "0.1"
lancedb = { path = "../rust/lancedb", default-features = false }
env_logger.workspace = true
pyo3 = { version = "0.24", features = ["extension-module", "abi3-py39"] }
@@ -34,6 +33,6 @@ pyo3-build-config = { version = "0.24", features = [
] }
[features]
default = ["remote", "lancedb/default"]
default = ["remote"]
fp16kernels = ["lancedb/fp16kernels"]
remote = ["lancedb/remote"]

View File

@@ -10,7 +10,6 @@ dependencies = [
"pyarrow>=16",
"pydantic>=1.10",
"tqdm>=4.27.0",
"lance-namespace==0.0.6"
]
description = "lancedb"
authors = [{ name = "LanceDB Devs", email = "dev@lancedb.com" }]

View File

@@ -19,7 +19,6 @@ from .remote.db import RemoteDBConnection
from .schema import vector
from .table import AsyncTable
from ._lancedb import Session
from .namespace import connect_namespace, LanceNamespaceDBConnection
def connect(
@@ -222,7 +221,6 @@ async def connect_async(
__all__ = [
"connect",
"connect_async",
"connect_namespace",
"AsyncConnection",
"AsyncTable",
"URI",
@@ -230,7 +228,6 @@ __all__ = [
"vector",
"DBConnection",
"LanceDBConnection",
"LanceNamespaceDBConnection",
"RemoteDBConnection",
"Session",
"__version__",

View File

@@ -21,28 +21,14 @@ class Session:
class Connection(object):
uri: str
async def is_open(self): ...
async def close(self): ...
async def list_namespaces(
self,
namespace: List[str],
page_token: Optional[str],
limit: Optional[int],
) -> List[str]: ...
async def create_namespace(self, namespace: List[str]) -> None: ...
async def drop_namespace(self, namespace: List[str]) -> None: ...
async def table_names(
self,
namespace: List[str],
start_after: Optional[str],
limit: Optional[int],
self, start_after: Optional[str], limit: Optional[int]
) -> list[str]: ...
async def create_table(
self,
name: str,
mode: str,
data: pa.RecordBatchReader,
namespace: List[str] = [],
storage_options: Optional[Dict[str, str]] = None,
) -> Table: ...
async def create_empty_table(
@@ -50,34 +36,10 @@ class Connection(object):
name: str,
mode: str,
schema: pa.Schema,
namespace: List[str] = [],
storage_options: Optional[Dict[str, str]] = None,
) -> Table: ...
async def open_table(
self,
name: str,
namespace: List[str] = [],
storage_options: Optional[Dict[str, str]] = None,
index_cache_size: Optional[int] = None,
) -> Table: ...
async def clone_table(
self,
target_table_name: str,
source_uri: str,
target_namespace: List[str] = [],
source_version: Optional[int] = None,
source_tag: Optional[str] = None,
is_shallow: bool = True,
) -> Table: ...
async def rename_table(
self,
cur_name: str,
new_name: str,
cur_namespace: List[str] = [],
new_namespace: List[str] = [],
) -> None: ...
async def drop_table(self, name: str, namespace: List[str] = []) -> None: ...
async def drop_all_tables(self, namespace: List[str] = []) -> None: ...
async def rename_table(self, old_name: str, new_name: str) -> None: ...
async def drop_table(self, name: str) -> None: ...
class Table:
def name(self) -> str: ...
@@ -97,10 +59,6 @@ class Table:
column: str,
index: Union[IvfFlat, IvfPq, HnswPq, HnswSq, BTree, Bitmap, LabelList, FTS],
replace: Optional[bool],
wait_timeout: Optional[object],
*,
name: Optional[str],
train: Optional[bool],
): ...
async def list_versions(self) -> List[Dict[str, Any]]: ...
async def version(self) -> int: ...

View File

@@ -43,70 +43,14 @@ if TYPE_CHECKING:
class DBConnection(EnforceOverrides):
"""An active LanceDB connection interface."""
def list_namespaces(
self,
namespace: List[str] = [],
page_token: Optional[str] = None,
limit: int = 10,
) -> Iterable[str]:
"""List immediate child namespace names in the given namespace.
Parameters
----------
namespace: List[str], default []
The parent namespace to list namespaces in.
Empty list represents root namespace.
page_token: str, optional
The token to use for pagination. If not present, start from the beginning.
limit: int, default 10
The size of the page to return.
Returns
-------
Iterable of str
List of immediate child namespace names
"""
return []
def create_namespace(self, namespace: List[str]) -> None:
"""Create a new namespace.
Parameters
----------
namespace: List[str]
The namespace identifier to create.
"""
raise NotImplementedError(
"Namespace operations are not supported for this connection type"
)
def drop_namespace(self, namespace: List[str]) -> None:
"""Drop a namespace.
Parameters
----------
namespace: List[str]
The namespace identifier to drop.
"""
raise NotImplementedError(
"Namespace operations are not supported for this connection type"
)
@abstractmethod
def table_names(
self,
page_token: Optional[str] = None,
limit: int = 10,
*,
namespace: List[str] = [],
self, page_token: Optional[str] = None, limit: int = 10
) -> Iterable[str]:
"""List all tables in this database, in sorted order
Parameters
----------
namespace: List[str], default []
The namespace to list tables in.
Empty list represents root namespace.
page_token: str, optional
The token to use for pagination. If not present, start from the beginning.
Typically, this token is last table name from the previous page.
@@ -133,7 +77,6 @@ class DBConnection(EnforceOverrides):
fill_value: float = 0.0,
embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None,
*,
namespace: List[str] = [],
storage_options: Optional[Dict[str, str]] = None,
data_storage_version: Optional[str] = None,
enable_v2_manifest_paths: Optional[bool] = None,
@@ -144,9 +87,6 @@ class DBConnection(EnforceOverrides):
----------
name: str
The name of the table.
namespace: List[str], default []
The namespace to create the table in.
Empty list represents root namespace.
data: The data to initialize the table, *optional*
User must provide at least one of `data` or `schema`.
Acceptable types are:
@@ -298,7 +238,6 @@ class DBConnection(EnforceOverrides):
self,
name: str,
*,
namespace: List[str] = [],
storage_options: Optional[Dict[str, str]] = None,
index_cache_size: Optional[int] = None,
) -> Table:
@@ -308,9 +247,6 @@ class DBConnection(EnforceOverrides):
----------
name: str
The name of the table.
namespace: List[str], optional
The namespace to open the table from.
None or empty list represents root namespace.
index_cache_size: int, default 256
**Deprecated**: Use session-level cache configuration instead.
Create a Session with custom cache sizes and pass it to lancedb.connect().
@@ -336,26 +272,17 @@ class DBConnection(EnforceOverrides):
"""
raise NotImplementedError
def drop_table(self, name: str, namespace: List[str] = []):
def drop_table(self, name: str):
"""Drop a table from the database.
Parameters
----------
name: str
The name of the table.
namespace: List[str], default []
The namespace to drop the table from.
Empty list represents root namespace.
"""
raise NotImplementedError
def rename_table(
self,
cur_name: str,
new_name: str,
cur_namespace: List[str] = [],
new_namespace: List[str] = [],
):
def rename_table(self, cur_name: str, new_name: str):
"""Rename a table in the database.
Parameters
@@ -364,12 +291,6 @@ class DBConnection(EnforceOverrides):
The current name of the table.
new_name: str
The new name of the table.
cur_namespace: List[str], optional
The namespace of the current table.
None or empty list represents root namespace.
new_namespace: List[str], optional
The namespace to move the table to.
If not specified, defaults to the same as cur_namespace.
"""
raise NotImplementedError
@@ -380,15 +301,9 @@ class DBConnection(EnforceOverrides):
"""
raise NotImplementedError
def drop_all_tables(self, namespace: List[str] = []):
def drop_all_tables(self):
"""
Drop all tables from the database
Parameters
----------
namespace: List[str], optional
The namespace to drop all tables from.
None or empty list represents root namespace.
"""
raise NotImplementedError
@@ -489,87 +404,18 @@ class LanceDBConnection(DBConnection):
conn = AsyncConnection(await lancedb_connect(self.uri))
return await conn.table_names(start_after=start_after, limit=limit)
@override
def list_namespaces(
self,
namespace: List[str] = [],
page_token: Optional[str] = None,
limit: int = 10,
) -> Iterable[str]:
"""List immediate child namespace names in the given namespace.
Parameters
----------
namespace: List[str], optional
The parent namespace to list namespaces in.
None or empty list represents root namespace.
page_token: str, optional
The token to use for pagination. If not present, start from the beginning.
limit: int, default 10
The size of the page to return.
Returns
-------
Iterable of str
List of immediate child namespace names
"""
return LOOP.run(
self._conn.list_namespaces(
namespace=namespace, page_token=page_token, limit=limit
)
)
@override
def create_namespace(self, namespace: List[str]) -> None:
"""Create a new namespace.
Parameters
----------
namespace: List[str]
The namespace identifier to create.
"""
LOOP.run(self._conn.create_namespace(namespace=namespace))
@override
def drop_namespace(self, namespace: List[str]) -> None:
"""Drop a namespace.
Parameters
----------
namespace: List[str]
The namespace identifier to drop.
"""
return LOOP.run(self._conn.drop_namespace(namespace=namespace))
@override
def table_names(
self,
page_token: Optional[str] = None,
limit: int = 10,
*,
namespace: List[str] = [],
self, page_token: Optional[str] = None, limit: int = 10
) -> Iterable[str]:
"""Get the names of all tables in the database. The names are sorted.
Parameters
----------
namespace: List[str], optional
The namespace to list tables in.
page_token: str, optional
The token to use for pagination.
limit: int, default 10
The maximum number of tables to return.
Returns
-------
Iterator of str.
A list of table names.
"""
return LOOP.run(
self._conn.table_names(
namespace=namespace, start_after=page_token, limit=limit
)
)
return LOOP.run(self._conn.table_names(start_after=page_token, limit=limit))
def __len__(self) -> int:
return len(self.table_names())
@@ -589,18 +435,12 @@ class LanceDBConnection(DBConnection):
fill_value: float = 0.0,
embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None,
*,
namespace: List[str] = [],
storage_options: Optional[Dict[str, str]] = None,
data_storage_version: Optional[str] = None,
enable_v2_manifest_paths: Optional[bool] = None,
) -> LanceTable:
"""Create a table in the database.
Parameters
----------
namespace: List[str], optional
The namespace to create the table in.
See
---
DBConnection.create_table
@@ -619,7 +459,6 @@ class LanceDBConnection(DBConnection):
on_bad_vectors=on_bad_vectors,
fill_value=fill_value,
embedding_functions=embedding_functions,
namespace=namespace,
storage_options=storage_options,
)
return tbl
@@ -629,7 +468,6 @@ class LanceDBConnection(DBConnection):
self,
name: str,
*,
namespace: List[str] = [],
storage_options: Optional[Dict[str, str]] = None,
index_cache_size: Optional[int] = None,
) -> LanceTable:
@@ -639,8 +477,6 @@ class LanceDBConnection(DBConnection):
----------
name: str
The name of the table.
namespace: List[str], optional
The namespace to open the table from.
Returns
-------
@@ -660,122 +496,26 @@ class LanceDBConnection(DBConnection):
return LanceTable.open(
self,
name,
namespace=namespace,
storage_options=storage_options,
index_cache_size=index_cache_size,
)
def clone_table(
self,
target_table_name: str,
source_uri: str,
*,
target_namespace: List[str] = [],
source_version: Optional[int] = None,
source_tag: Optional[str] = None,
is_shallow: bool = True,
) -> LanceTable:
"""Clone a table from a source table.
A shallow clone creates a new table that shares the underlying data files
with the source table but has its own independent manifest. This allows
both the source and cloned tables to evolve independently while initially
sharing the same data, deletion, and index files.
Parameters
----------
target_table_name: str
The name of the target table to create.
source_uri: str
The URI of the source table to clone from.
target_namespace: List[str], optional
The namespace for the target table.
None or empty list represents root namespace.
source_version: int, optional
The version of the source table to clone.
source_tag: str, optional
The tag of the source table to clone.
is_shallow: bool, default True
Whether to perform a shallow clone (True) or deep clone (False).
Currently only shallow clone is supported.
Returns
-------
A LanceTable object representing the cloned table.
"""
LOOP.run(
self._conn.clone_table(
target_table_name,
source_uri,
target_namespace=target_namespace,
source_version=source_version,
source_tag=source_tag,
is_shallow=is_shallow,
)
)
return LanceTable.open(
self,
target_table_name,
namespace=target_namespace,
)
@override
def drop_table(
self,
name: str,
namespace: List[str] = [],
ignore_missing: bool = False,
):
def drop_table(self, name: str, ignore_missing: bool = False):
"""Drop a table from the database.
Parameters
----------
name: str
The name of the table.
namespace: List[str], optional
The namespace to drop the table from.
ignore_missing: bool, default False
If True, ignore if the table does not exist.
"""
LOOP.run(
self._conn.drop_table(
name, namespace=namespace, ignore_missing=ignore_missing
)
)
LOOP.run(self._conn.drop_table(name, ignore_missing=ignore_missing))
@override
def drop_all_tables(self, namespace: List[str] = []):
LOOP.run(self._conn.drop_all_tables(namespace=namespace))
@override
def rename_table(
self,
cur_name: str,
new_name: str,
cur_namespace: List[str] = [],
new_namespace: List[str] = [],
):
"""Rename a table in the database.
Parameters
----------
cur_name: str
The current name of the table.
new_name: str
The new name of the table.
cur_namespace: List[str], optional
The namespace of the current table.
new_namespace: List[str], optional
The namespace to move the table to.
"""
LOOP.run(
self._conn.rename_table(
cur_name,
new_name,
cur_namespace=cur_namespace,
new_namespace=new_namespace,
)
)
def drop_all_tables(self):
LOOP.run(self._conn.drop_all_tables())
@deprecation.deprecated(
deprecated_in="0.15.1",
@@ -848,67 +588,13 @@ class AsyncConnection(object):
def uri(self) -> str:
return self._inner.uri
async def list_namespaces(
self,
namespace: List[str] = [],
page_token: Optional[str] = None,
limit: int = 10,
) -> Iterable[str]:
"""List immediate child namespace names in the given namespace.
Parameters
----------
namespace: List[str], optional
The parent namespace to list namespaces in.
None or empty list represents root namespace.
page_token: str, optional
The token to use for pagination. If not present, start from the beginning.
limit: int, default 10
The size of the page to return.
Returns
-------
Iterable of str
List of immediate child namespace names (not full paths)
"""
return await self._inner.list_namespaces(
namespace=namespace, page_token=page_token, limit=limit
)
async def create_namespace(self, namespace: List[str]) -> None:
"""Create a new namespace.
Parameters
----------
namespace: List[str]
The namespace identifier to create.
"""
await self._inner.create_namespace(namespace)
async def drop_namespace(self, namespace: List[str]) -> None:
"""Drop a namespace.
Parameters
----------
namespace: List[str]
The namespace identifier to drop.
"""
await self._inner.drop_namespace(namespace)
async def table_names(
self,
*,
namespace: List[str] = [],
start_after: Optional[str] = None,
limit: Optional[int] = None,
self, *, start_after: Optional[str] = None, limit: Optional[int] = None
) -> Iterable[str]:
"""List all tables in this database, in sorted order
Parameters
----------
namespace: List[str], optional
The namespace to list tables in.
None or empty list represents root namespace.
start_after: str, optional
If present, only return names that come lexicographically after the supplied
value.
@@ -922,9 +608,7 @@ class AsyncConnection(object):
-------
Iterable of str
"""
return await self._inner.table_names(
namespace=namespace, start_after=start_after, limit=limit
)
return await self._inner.table_names(start_after=start_after, limit=limit)
async def create_table(
self,
@@ -937,7 +621,6 @@ class AsyncConnection(object):
fill_value: Optional[float] = None,
storage_options: Optional[Dict[str, str]] = None,
*,
namespace: List[str] = [],
embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None,
) -> AsyncTable:
"""Create an [AsyncTable][lancedb.table.AsyncTable] in the database.
@@ -946,9 +629,6 @@ class AsyncConnection(object):
----------
name: str
The name of the table.
namespace: List[str], default []
The namespace to create the table in.
Empty list represents root namespace.
data: The data to initialize the table, *optional*
User must provide at least one of `data` or `schema`.
Acceptable types are:
@@ -1127,7 +807,6 @@ class AsyncConnection(object):
name,
mode,
schema,
namespace=namespace,
storage_options=storage_options,
)
else:
@@ -1136,7 +815,6 @@ class AsyncConnection(object):
name,
mode,
data,
namespace=namespace,
storage_options=storage_options,
)
@@ -1145,8 +823,6 @@ class AsyncConnection(object):
async def open_table(
self,
name: str,
*,
namespace: List[str] = [],
storage_options: Optional[Dict[str, str]] = None,
index_cache_size: Optional[int] = None,
) -> AsyncTable:
@@ -1156,9 +832,6 @@ class AsyncConnection(object):
----------
name: str
The name of the table.
namespace: List[str], optional
The namespace to open the table from.
None or empty list represents root namespace.
storage_options: dict, optional
Additional options for the storage backend. Options already set on the
connection will be inherited by the table, but can be overridden here.
@@ -1182,125 +855,42 @@ class AsyncConnection(object):
-------
A LanceTable object representing the table.
"""
table = await self._inner.open_table(
name,
namespace=namespace,
storage_options=storage_options,
index_cache_size=index_cache_size,
)
table = await self._inner.open_table(name, storage_options, index_cache_size)
return AsyncTable(table)
async def clone_table(
self,
target_table_name: str,
source_uri: str,
*,
target_namespace: List[str] = [],
source_version: Optional[int] = None,
source_tag: Optional[str] = None,
is_shallow: bool = True,
) -> AsyncTable:
"""Clone a table from a source table.
A shallow clone creates a new table that shares the underlying data files
with the source table but has its own independent manifest. This allows
both the source and cloned tables to evolve independently while initially
sharing the same data, deletion, and index files.
Parameters
----------
target_table_name: str
The name of the target table to create.
source_uri: str
The URI of the source table to clone from.
target_namespace: List[str], optional
The namespace for the target table.
None or empty list represents root namespace.
source_version: int, optional
The version of the source table to clone.
source_tag: str, optional
The tag of the source table to clone.
is_shallow: bool, default True
Whether to perform a shallow clone (True) or deep clone (False).
Currently only shallow clone is supported.
Returns
-------
An AsyncTable object representing the cloned table.
"""
table = await self._inner.clone_table(
target_table_name,
source_uri,
target_namespace=target_namespace,
source_version=source_version,
source_tag=source_tag,
is_shallow=is_shallow,
)
return AsyncTable(table)
async def rename_table(
self,
cur_name: str,
new_name: str,
cur_namespace: List[str] = [],
new_namespace: List[str] = [],
):
async def rename_table(self, old_name: str, new_name: str):
"""Rename a table in the database.
Parameters
----------
cur_name: str
old_name: str
The current name of the table.
new_name: str
The new name of the table.
cur_namespace: List[str], optional
The namespace of the current table.
None or empty list represents root namespace.
new_namespace: List[str], optional
The namespace to move the table to.
If not specified, defaults to the same as cur_namespace.
"""
await self._inner.rename_table(
cur_name, new_name, cur_namespace=cur_namespace, new_namespace=new_namespace
)
await self._inner.rename_table(old_name, new_name)
async def drop_table(
self,
name: str,
*,
namespace: List[str] = [],
ignore_missing: bool = False,
):
async def drop_table(self, name: str, *, ignore_missing: bool = False):
"""Drop a table from the database.
Parameters
----------
name: str
The name of the table.
namespace: List[str], default []
The namespace to drop the table from.
Empty list represents root namespace.
ignore_missing: bool, default False
If True, ignore if the table does not exist.
"""
try:
await self._inner.drop_table(name, namespace=namespace)
await self._inner.drop_table(name)
except ValueError as e:
if not ignore_missing:
raise e
if f"Table '{name}' was not found" not in str(e):
raise e
async def drop_all_tables(self, namespace: List[str] = []):
"""Drop all tables from the database.
Parameters
----------
namespace: List[str], optional
The namespace to drop all tables from.
None or empty list represents root namespace.
"""
await self._inner.drop_all_tables(namespace=namespace)
async def drop_all_tables(self):
"""Drop all tables from the database."""
await self._inner.drop_all_tables()
@deprecation.deprecated(
deprecated_in="0.15.1",

View File

@@ -122,7 +122,7 @@ class EmbeddingFunctionRegistry:
obj["vector_column"]: EmbeddingFunctionConfig(
vector_column=obj["vector_column"],
source_column=obj["source_column"],
function=self.get(obj["name"]).create(**obj["model"]),
function=self.get(obj["name"])(**obj["model"]),
)
for obj in raw_list
}

View File

@@ -251,13 +251,6 @@ class HnswPq:
results. In most cases, there is no benefit to setting this higher than 500.
This value should be set to a value that is not less than `ef` in the
search phase.
target_partition_size, default is 1,048,576
The target size of each partition.
This value controls the tradeoff between search performance and accuracy.
faster search but less accurate results as higher value.
"""
distance_type: Literal["l2", "cosine", "dot"] = "l2"
@@ -268,7 +261,6 @@ class HnswPq:
sample_rate: int = 256
m: int = 20
ef_construction: int = 300
target_partition_size: Optional[int] = None
@dataclass
@@ -359,12 +351,6 @@ class HnswSq:
This value should be set to a value that is not less than `ef` in the search
phase.
target_partition_size, default is 1,048,576
The target size of each partition.
This value controls the tradeoff between search performance and accuracy.
faster search but less accurate results as higher value.
"""
distance_type: Literal["l2", "cosine", "dot"] = "l2"
@@ -373,7 +359,6 @@ class HnswSq:
sample_rate: int = 256
m: int = 20
ef_construction: int = 300
target_partition_size: Optional[int] = None
@dataclass
@@ -459,20 +444,12 @@ class IvfFlat:
cases the default should be sufficient.
The default value is 256.
target_partition_size, default is 8192
The target size of each partition.
This value controls the tradeoff between search performance and accuracy.
faster search but less accurate results as higher value.
"""
distance_type: Literal["l2", "cosine", "dot", "hamming"] = "l2"
num_partitions: Optional[int] = None
max_iterations: int = 50
sample_rate: int = 256
target_partition_size: Optional[int] = None
@dataclass
@@ -587,13 +564,6 @@ class IvfPq:
cases the default should be sufficient.
The default value is 256.
target_partition_size, default is 8192
The target size of each partition.
This value controls the tradeoff between search performance and accuracy.
faster search but less accurate results as higher value.
"""
distance_type: Literal["l2", "cosine", "dot"] = "l2"
@@ -602,7 +572,6 @@ class IvfPq:
num_bits: int = 8
max_iterations: int = 50
sample_rate: int = 256
target_partition_size: Optional[int] = None
__all__ = [

View File

@@ -1,401 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""
LanceDB Namespace integration module.
This module provides integration with lance_namespace for managing tables
through a namespace abstraction.
"""
from __future__ import annotations
from typing import Dict, Iterable, List, Optional, Union
import os
from lancedb.db import DBConnection
from lancedb.table import LanceTable, Table
from lancedb.util import validate_table_name
from lancedb.common import validate_schema
from lancedb.table import sanitize_create_table
from overrides import override
from lance_namespace import LanceNamespace, connect as namespace_connect
from lance_namespace_urllib3_client.models import (
ListTablesRequest,
DescribeTableRequest,
CreateTableRequest,
DropTableRequest,
ListNamespacesRequest,
CreateNamespaceRequest,
DropNamespaceRequest,
JsonArrowSchema,
JsonArrowField,
JsonArrowDataType,
)
import pyarrow as pa
from datetime import timedelta
from lancedb.pydantic import LanceModel
from lancedb.common import DATA
from lancedb.embeddings import EmbeddingFunctionConfig
from ._lancedb import Session
def _convert_pyarrow_type_to_json(arrow_type: pa.DataType) -> JsonArrowDataType:
"""Convert PyArrow DataType to JsonArrowDataType."""
if pa.types.is_null(arrow_type):
type_name = "null"
elif pa.types.is_boolean(arrow_type):
type_name = "bool"
elif pa.types.is_int8(arrow_type):
type_name = "int8"
elif pa.types.is_uint8(arrow_type):
type_name = "uint8"
elif pa.types.is_int16(arrow_type):
type_name = "int16"
elif pa.types.is_uint16(arrow_type):
type_name = "uint16"
elif pa.types.is_int32(arrow_type):
type_name = "int32"
elif pa.types.is_uint32(arrow_type):
type_name = "uint32"
elif pa.types.is_int64(arrow_type):
type_name = "int64"
elif pa.types.is_uint64(arrow_type):
type_name = "uint64"
elif pa.types.is_float32(arrow_type):
type_name = "float32"
elif pa.types.is_float64(arrow_type):
type_name = "float64"
elif pa.types.is_string(arrow_type):
type_name = "utf8"
elif pa.types.is_binary(arrow_type):
type_name = "binary"
elif pa.types.is_list(arrow_type):
# For list types, we need more complex handling
type_name = "list"
elif pa.types.is_fixed_size_list(arrow_type):
type_name = "fixed_size_list"
else:
# Default to string representation for unsupported types
type_name = str(arrow_type)
return JsonArrowDataType(type=type_name)
def _convert_pyarrow_schema_to_json(schema: pa.Schema) -> JsonArrowSchema:
"""Convert PyArrow Schema to JsonArrowSchema."""
fields = []
for field in schema:
json_field = JsonArrowField(
name=field.name,
type=_convert_pyarrow_type_to_json(field.type),
nullable=field.nullable,
metadata=field.metadata,
)
fields.append(json_field)
return JsonArrowSchema(fields=fields, metadata=schema.metadata)
class LanceNamespaceDBConnection(DBConnection):
"""
A LanceDB connection that uses a namespace for table management.
This connection delegates table URI resolution to a lance_namespace instance,
while using the standard LanceTable for actual table operations.
"""
def __init__(
self,
namespace: LanceNamespace,
*,
read_consistency_interval: Optional[timedelta] = None,
storage_options: Optional[Dict[str, str]] = None,
session: Optional[Session] = None,
):
"""
Initialize a namespace-based LanceDB connection.
Parameters
----------
namespace : LanceNamespace
The namespace instance to use for table management
read_consistency_interval : Optional[timedelta]
The interval at which to check for updates to the table from other
processes. If None, then consistency is not checked.
storage_options : Optional[Dict[str, str]]
Additional options for the storage backend
session : Optional[Session]
A session to use for this connection
"""
self._ns = namespace
self.read_consistency_interval = read_consistency_interval
self.storage_options = storage_options or {}
self.session = session
@override
def table_names(
self,
page_token: Optional[str] = None,
limit: int = 10,
*,
namespace: List[str] = [],
) -> Iterable[str]:
request = ListTablesRequest(id=namespace, page_token=page_token, limit=limit)
response = self._ns.list_tables(request)
return response.tables if response.tables else []
@override
def create_table(
self,
name: str,
data: Optional[DATA] = None,
schema: Optional[Union[pa.Schema, LanceModel]] = None,
mode: str = "create",
exist_ok: bool = False,
on_bad_vectors: str = "error",
fill_value: float = 0.0,
embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None,
*,
namespace: List[str] = [],
storage_options: Optional[Dict[str, str]] = None,
data_storage_version: Optional[str] = None,
enable_v2_manifest_paths: Optional[bool] = None,
) -> Table:
if mode.lower() not in ["create", "overwrite"]:
raise ValueError("mode must be either 'create' or 'overwrite'")
validate_table_name(name)
# TODO: support passing data
if data is not None:
raise ValueError(
"create_table currently only supports creating empty tables (data=None)"
)
# Prepare schema
metadata = None
if embedding_functions is not None:
from lancedb.embeddings.registry import EmbeddingFunctionRegistry
registry = EmbeddingFunctionRegistry.get_instance()
metadata = registry.get_table_metadata(embedding_functions)
data, schema = sanitize_create_table(
data, schema, metadata, on_bad_vectors, fill_value
)
validate_schema(schema)
# Convert PyArrow schema to JsonArrowSchema
json_schema = _convert_pyarrow_schema_to_json(schema)
# Create table request with namespace
table_id = namespace + [name]
request = CreateTableRequest(id=table_id, var_schema=json_schema)
# Create empty Arrow IPC stream bytes
import pyarrow.ipc as ipc
import io
empty_table = pa.Table.from_arrays(
[pa.array([], type=field.type) for field in schema], schema=schema
)
buffer = io.BytesIO()
with ipc.new_stream(buffer, schema) as writer:
writer.write_table(empty_table)
request_data = buffer.getvalue()
self._ns.create_table(request, request_data)
return self.open_table(
name, namespace=namespace, storage_options=storage_options
)
@override
def open_table(
self,
name: str,
*,
namespace: List[str] = [],
storage_options: Optional[Dict[str, str]] = None,
index_cache_size: Optional[int] = None,
) -> Table:
table_id = namespace + [name]
request = DescribeTableRequest(id=table_id)
response = self._ns.describe_table(request)
merged_storage_options = dict()
if storage_options:
merged_storage_options.update(storage_options)
if response.storage_options:
merged_storage_options.update(response.storage_options)
return self._lance_table_from_uri(
response.location,
storage_options=merged_storage_options,
index_cache_size=index_cache_size,
)
@override
def drop_table(self, name: str, namespace: List[str] = []):
# Use namespace drop_table directly
table_id = namespace + [name]
request = DropTableRequest(id=table_id)
self._ns.drop_table(request)
@override
def rename_table(
self,
cur_name: str,
new_name: str,
cur_namespace: List[str] = [],
new_namespace: List[str] = [],
):
raise NotImplementedError(
"rename_table is not supported for namespace connections"
)
@override
def drop_database(self):
raise NotImplementedError(
"drop_database is deprecated, use drop_all_tables instead"
)
@override
def drop_all_tables(self, namespace: List[str] = []):
for table_name in self.table_names(namespace=namespace):
self.drop_table(table_name, namespace=namespace)
@override
def list_namespaces(
self,
namespace: List[str] = [],
page_token: Optional[str] = None,
limit: int = 10,
) -> Iterable[str]:
"""
List child namespaces under the given namespace.
Parameters
----------
namespace : Optional[List[str]]
The parent namespace to list children from.
If None, lists root-level namespaces.
page_token : Optional[str]
Pagination token for listing results.
limit : int
Maximum number of namespaces to return.
Returns
-------
Iterable[str]
Names of child namespaces.
"""
request = ListNamespacesRequest(
id=namespace, page_token=page_token, limit=limit
)
response = self._ns.list_namespaces(request)
return response.namespaces if response.namespaces else []
@override
def create_namespace(self, namespace: List[str]) -> None:
"""
Create a new namespace.
Parameters
----------
namespace : List[str]
The namespace path to create.
"""
request = CreateNamespaceRequest(id=namespace)
self._ns.create_namespace(request)
@override
def drop_namespace(self, namespace: List[str]) -> None:
"""
Drop a namespace.
Parameters
----------
namespace : List[str]
The namespace path to drop.
"""
request = DropNamespaceRequest(id=namespace)
self._ns.drop_namespace(request)
def _lance_table_from_uri(
self,
table_uri: str,
*,
storage_options: Optional[Dict[str, str]] = None,
index_cache_size: Optional[int] = None,
) -> LanceTable:
# Extract the base path and table name from the URI
if table_uri.endswith(".lance"):
base_path = os.path.dirname(table_uri)
table_name = os.path.basename(table_uri)[:-6] # Remove .lance
else:
raise ValueError(f"Invalid table URI: {table_uri}")
from lancedb.db import LanceDBConnection
temp_conn = LanceDBConnection(
base_path,
read_consistency_interval=self.read_consistency_interval,
storage_options={**self.storage_options, **(storage_options or {})},
session=self.session,
)
# Open the table using the temporary connection
return LanceTable.open(
temp_conn,
table_name,
storage_options=storage_options,
index_cache_size=index_cache_size,
)
def connect_namespace(
impl: str,
properties: Dict[str, str],
*,
read_consistency_interval: Optional[timedelta] = None,
storage_options: Optional[Dict[str, str]] = None,
session: Optional[Session] = None,
) -> LanceNamespaceDBConnection:
"""
Connect to a LanceDB database through a namespace.
Parameters
----------
impl : str
The namespace implementation to use. For examples:
- "dir" for DirectoryNamespace
- "rest" for REST-based namespace
- Full module path for custom implementations
properties : Dict[str, str]
Configuration properties for the namespace implementation.
Different namespace implemenation has different config properties.
For example, use DirectoryNamespace with {"root": "/path/to/directory"}
read_consistency_interval : Optional[timedelta]
The interval at which to check for updates to the table from other
processes. If None, then consistency is not checked.
storage_options : Optional[Dict[str, str]]
Additional options for the storage backend
session : Optional[Session]
A session to use for this connection
Returns
-------
LanceNamespaceDBConnection
A namespace-based connection to LanceDB
"""
namespace = namespace_connect(impl, properties)
# Return the namespace-based connection
return LanceNamespaceDBConnection(
namespace,
read_consistency_interval=read_consistency_interval,
storage_options=storage_options,
session=session,
)

View File

@@ -28,7 +28,6 @@ import pyarrow.fs as pa_fs
import pydantic
from lancedb.pydantic import PYDANTIC_VERSION
from lancedb.background_loop import LOOP
from . import __version__
from .arrow import AsyncRecordBatchReader
@@ -49,7 +48,6 @@ if TYPE_CHECKING:
from ._lancedb import FTSQuery as LanceFTSQuery
from ._lancedb import HybridQuery as LanceHybridQuery
from ._lancedb import VectorQuery as LanceVectorQuery
from ._lancedb import TakeQuery as LanceTakeQuery
from ._lancedb import PyQueryRequest
from .common import VEC
from .pydantic import LanceModel
@@ -912,7 +910,7 @@ class LanceQueryBuilder(ABC):
ProjectionExec: expr=[vector@0 as vector, _distance@2 as _distance]
GlobalLimitExec: skip=0, fetch=10
FilterExec: _distance@2 IS NOT NULL
SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], preserve_partitioning=[false]
KNNVectorDistance: metric=l2
LanceRead: uri=..., projection=[vector], ...
@@ -943,22 +941,20 @@ class LanceQueryBuilder(ABC):
>>> query = [100, 100]
>>> plan = table.search(query).analyze_plan()
>>> print(plan) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
AnalyzeExec verbose=true, metrics=[], cumulative_cpu=...
TracedExec, metrics=[], cumulative_cpu=...
ProjectionExec: expr=[...], metrics=[...], cumulative_cpu=...
GlobalLimitExec: skip=0, fetch=10, metrics=[...], cumulative_cpu=...
AnalyzeExec verbose=true, metrics=[]
TracedExec, metrics=[]
ProjectionExec: expr=[...], metrics=[...]
GlobalLimitExec: skip=0, fetch=10, metrics=[...]
FilterExec: _distance@2 IS NOT NULL,
metrics=[output_rows=..., elapsed_compute=...], cumulative_cpu=...
metrics=[output_rows=..., elapsed_compute=...]
SortExec: TopK(fetch=10), expr=[...],
preserve_partitioning=[...],
metrics=[output_rows=..., elapsed_compute=..., row_replacements=...],
cumulative_cpu=...
metrics=[output_rows=..., elapsed_compute=..., row_replacements=...]
KNNVectorDistance: metric=l2,
metrics=[output_rows=..., elapsed_compute=..., output_batches=...],
cumulative_cpu=...
metrics=[output_rows=..., elapsed_compute=..., output_batches=...]
LanceRead: uri=..., projection=[vector], ...
metrics=[output_rows=..., elapsed_compute=...,
bytes_read=..., iops=..., requests=...], cumulative_cpu=...
bytes_read=..., iops=..., requests=...]
Returns
-------
@@ -2045,11 +2041,11 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
>>> plan = table.search(query).explain_plan(True)
>>> print(plan) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
ProjectionExec: expr=[vector@0 as vector, _distance@2 as _distance]
GlobalLimitExec: skip=0, fetch=10
FilterExec: _distance@2 IS NOT NULL
SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
KNNVectorDistance: metric=l2
LanceRead: uri=..., projection=[vector], ...
GlobalLimitExec: skip=0, fetch=10
FilterExec: _distance@2 IS NOT NULL
SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], preserve_partitioning=[false]
KNNVectorDistance: metric=l2
LanceRead: uri=..., projection=[vector], ...
Parameters
----------
@@ -2143,11 +2139,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
class AsyncQueryBase(object):
"""
Base class for all async queries (take, scan, vector, fts, hybrid)
"""
def __init__(self, inner: Union[LanceQuery, LanceVectorQuery, LanceTakeQuery]):
def __init__(self, inner: Union[LanceQuery, LanceVectorQuery]):
"""
Construct an AsyncQueryBase
@@ -2157,14 +2149,27 @@ class AsyncQueryBase(object):
self._inner = inner
def to_query_object(self) -> Query:
"""
Convert the query into a query object
This is currently experimental but can be useful as the query object is pure
python and more easily serializable.
"""
return Query.from_inner(self._inner.to_query_request())
def where(self, predicate: str) -> Self:
"""
Only return rows matching the given predicate
The predicate should be supplied as an SQL query string.
Examples
--------
>>> predicate = "x > 10"
>>> predicate = "y > 0 AND y < 100"
>>> predicate = "x > 5 OR y = 'test'"
Filtering performance can often be improved by creating a scalar index
on the filter column(s).
"""
self._inner.where(predicate)
return self
def select(self, columns: Union[List[str], dict[str, str]]) -> Self:
"""
Return only the specified columns.
@@ -2203,6 +2208,42 @@ class AsyncQueryBase(object):
raise TypeError("columns must be a list of column names or a dict")
return self
def limit(self, limit: int) -> Self:
"""
Set the maximum number of results to return.
By default, a plain search has no limit. If this method is not
called then every valid row from the table will be returned.
"""
self._inner.limit(limit)
return self
def offset(self, offset: int) -> Self:
"""
Set the offset for the results.
Parameters
----------
offset: int
The offset to start fetching results from.
"""
self._inner.offset(offset)
return self
def fast_search(self) -> Self:
"""
Skip searching un-indexed data.
This can make queries faster, but will miss any data that has not been
indexed.
!!! tip
You can add new data into an existing index by calling
[AsyncTable.optimize][lancedb.table.AsyncTable.optimize].
"""
self._inner.fast_search()
return self
def with_row_id(self) -> Self:
"""
Include the _rowid column in the results.
@@ -2210,6 +2251,27 @@ class AsyncQueryBase(object):
self._inner.with_row_id()
return self
def postfilter(self) -> Self:
"""
If this is called then filtering will happen after the search instead of
before.
By default filtering will be performed before the search. This is how
filtering is typically understood to work. This prefilter step does add some
additional latency. Creating a scalar index on the filter column(s) can
often improve this latency. However, sometimes a filter is too complex or
scalar indices cannot be applied to the column. In these cases postfiltering
can be used instead of prefiltering to improve latency.
Post filtering applies the filter to the results of the search. This
means we only run the filter on a much smaller set of data. However, it can
cause the query to return fewer than `limit` results (or even no results) if
none of the nearest results match the filter.
Post filtering happens during the "refine stage" (described in more detail in
@see {@link VectorQuery#refineFactor}). This means that setting a higher refine
factor can often help restore some of the results lost by post filtering.
"""
self._inner.postfilter()
return self
async def to_batches(
self,
*,
@@ -2233,9 +2295,7 @@ class AsyncQueryBase(object):
complete within the specified time, an error will be raised.
"""
return AsyncRecordBatchReader(
await self._inner.execute(
max_batch_length=max_batch_length, timeout=timeout
)
await self._inner.execute(max_batch_length, timeout)
)
async def to_arrow(self, timeout: Optional[timedelta] = None) -> pa.Table:
@@ -2369,7 +2429,7 @@ class AsyncQueryBase(object):
ProjectionExec: expr=[vector@0 as vector, _distance@2 as _distance]
GlobalLimitExec: skip=0, fetch=10
FilterExec: _distance@2 IS NOT NULL
SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], preserve_partitioning=[false]
KNNVectorDistance: metric=l2
LanceRead: uri=..., projection=[vector], ...
@@ -2394,98 +2454,7 @@ class AsyncQueryBase(object):
return await self._inner.analyze_plan()
class AsyncStandardQuery(AsyncQueryBase):
"""
Base class for "standard" async queries (all but take currently)
"""
def __init__(self, inner: Union[LanceQuery, LanceVectorQuery]):
"""
Construct an AsyncStandardQuery
This method is not intended to be called directly. Instead, use the
[AsyncTable.query][lancedb.table.AsyncTable.query] method to create a query.
"""
super().__init__(inner)
def where(self, predicate: str) -> Self:
"""
Only return rows matching the given predicate
The predicate should be supplied as an SQL query string.
Examples
--------
>>> predicate = "x > 10"
>>> predicate = "y > 0 AND y < 100"
>>> predicate = "x > 5 OR y = 'test'"
Filtering performance can often be improved by creating a scalar index
on the filter column(s).
"""
self._inner.where(predicate)
return self
def limit(self, limit: int) -> Self:
"""
Set the maximum number of results to return.
By default, a plain search has no limit. If this method is not
called then every valid row from the table will be returned.
"""
self._inner.limit(limit)
return self
def offset(self, offset: int) -> Self:
"""
Set the offset for the results.
Parameters
----------
offset: int
The offset to start fetching results from.
"""
self._inner.offset(offset)
return self
def fast_search(self) -> Self:
"""
Skip searching un-indexed data.
This can make queries faster, but will miss any data that has not been
indexed.
!!! tip
You can add new data into an existing index by calling
[AsyncTable.optimize][lancedb.table.AsyncTable.optimize].
"""
self._inner.fast_search()
return self
def postfilter(self) -> Self:
"""
If this is called then filtering will happen after the search instead of
before.
By default filtering will be performed before the search. This is how
filtering is typically understood to work. This prefilter step does add some
additional latency. Creating a scalar index on the filter column(s) can
often improve this latency. However, sometimes a filter is too complex or
scalar indices cannot be applied to the column. In these cases postfiltering
can be used instead of prefiltering to improve latency.
Post filtering applies the filter to the results of the search. This
means we only run the filter on a much smaller set of data. However, it can
cause the query to return fewer than `limit` results (or even no results) if
none of the nearest results match the filter.
Post filtering happens during the "refine stage" (described in more detail in
@see {@link VectorQuery#refineFactor}). This means that setting a higher refine
factor can often help restore some of the results lost by post filtering.
"""
self._inner.postfilter()
return self
class AsyncQuery(AsyncStandardQuery):
class AsyncQuery(AsyncQueryBase):
def __init__(self, inner: LanceQuery):
"""
Construct an AsyncQuery
@@ -2619,7 +2588,7 @@ class AsyncQuery(AsyncStandardQuery):
return AsyncFTSQuery(self._inner.nearest_to_text({"query": query}))
class AsyncFTSQuery(AsyncStandardQuery):
class AsyncFTSQuery(AsyncQueryBase):
"""A query for full text search for LanceDB."""
def __init__(self, inner: LanceFTSQuery):
@@ -2898,7 +2867,7 @@ class AsyncVectorQueryBase:
return self
class AsyncVectorQuery(AsyncStandardQuery, AsyncVectorQueryBase):
class AsyncVectorQuery(AsyncQueryBase, AsyncVectorQueryBase):
def __init__(self, inner: LanceVectorQuery):
"""
Construct an AsyncVectorQuery
@@ -2981,7 +2950,7 @@ class AsyncVectorQuery(AsyncStandardQuery, AsyncVectorQueryBase):
return AsyncRecordBatchReader(results, max_batch_length=max_batch_length)
class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
class AsyncHybridQuery(AsyncQueryBase, AsyncVectorQueryBase):
"""
A query builder that performs hybrid vector and full text search.
Results are combined and reranked based on the specified reranker.
@@ -3085,7 +3054,7 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
CoalesceBatchesExec: target_batch_size=1024
GlobalLimitExec: skip=0, fetch=10
FilterExec: _distance@2 IS NOT NULL
SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], preserve_partitioning=[false]
KNNVectorDistance: metric=l2
LanceRead: uri=..., projection=[vector], ...
<BLANKLINE>
@@ -3133,252 +3102,3 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
results.append(await self._inner.to_fts_query().analyze_plan())
return "\n".join(results)
class AsyncTakeQuery(AsyncQueryBase):
"""
Builder for parameterizing and executing take queries.
"""
def __init__(self, inner: LanceTakeQuery):
super().__init__(inner)
class BaseQueryBuilder(object):
"""
Wraps AsyncQueryBase and provides a synchronous interface
"""
def __init__(self, inner: AsyncQueryBase):
self._inner = inner
def to_query_object(self) -> Query:
return self._inner.to_query_object()
def select(self, columns: Union[List[str], dict[str, str]]) -> Self:
"""
Return only the specified columns.
By default a query will return all columns from the table. However, this can
have a very significant impact on latency. LanceDb stores data in a columnar
fashion. This
means we can finely tune our I/O to select exactly the columns we need.
As a best practice you should always limit queries to the columns that you need.
If you pass in a list of column names then only those columns will be
returned.
You can also use this method to create new "dynamic" columns based on your
existing columns. For example, you may not care about "a" or "b" but instead
simply want "a + b". This is often seen in the SELECT clause of an SQL query
(e.g. `SELECT a+b FROM my_table`).
To create dynamic columns you can pass in a dict[str, str]. A column will be
returned for each entry in the map. The key provides the name of the column.
The value is an SQL string used to specify how the column is calculated.
For example, an SQL query might state `SELECT a + b AS combined, c`. The
equivalent input to this method would be `{"combined": "a + b", "c": "c"}`.
Columns will always be returned in the order given, even if that order is
different than the order used when adding the data.
"""
self._inner.select(columns)
return self
def with_row_id(self) -> Self:
"""
Include the _rowid column in the results.
"""
self._inner.with_row_id()
return self
def to_batches(
self,
*,
max_batch_length: Optional[int] = None,
timeout: Optional[timedelta] = None,
) -> pa.RecordBatchReader:
"""
Execute the query and return the results as an Apache Arrow RecordBatchReader.
Parameters
----------
max_batch_length: Optional[int]
The maximum number of selected records in a single RecordBatch object.
If not specified, a default batch length is used.
It is possible for batches to be smaller than the provided length if the
underlying data is stored in smaller chunks.
timeout: Optional[timedelta]
The maximum time to wait for the query to complete.
If not specified, no timeout is applied. If the query does not
complete within the specified time, an error will be raised.
"""
async_iter = LOOP.run(self._inner.execute(max_batch_length, timeout))
def iter_sync():
try:
while True:
yield LOOP.run(async_iter.__anext__())
except StopAsyncIteration:
return
return pa.RecordBatchReader.from_batches(async_iter.schema, iter_sync())
def to_arrow(self, timeout: Optional[timedelta] = None) -> pa.Table:
"""
Execute the query and collect the results into an Apache Arrow Table.
This method will collect all results into memory before returning. If
you expect a large number of results, you may want to use
[to_batches][lancedb.query.AsyncQueryBase.to_batches]
Parameters
----------
timeout: Optional[timedelta]
The maximum time to wait for the query to complete.
If not specified, no timeout is applied. If the query does not
complete within the specified time, an error will be raised.
"""
return LOOP.run(self._inner.to_arrow(timeout))
def to_list(self, timeout: Optional[timedelta] = None) -> List[dict]:
"""
Execute the query and return the results as a list of dictionaries.
Each list entry is a dictionary with the selected column names as keys,
or all table columns if `select` is not called. The vector and the "_distance"
fields are returned whether or not they're explicitly selected.
Parameters
----------
timeout: Optional[timedelta]
The maximum time to wait for the query to complete.
If not specified, no timeout is applied. If the query does not
complete within the specified time, an error will be raised.
"""
return LOOP.run(self._inner.to_list(timeout))
def to_pandas(
self,
flatten: Optional[Union[int, bool]] = None,
timeout: Optional[timedelta] = None,
) -> "pd.DataFrame":
"""
Execute the query and collect the results into a pandas DataFrame.
This method will collect all results into memory before returning. If you
expect a large number of results, you may want to use
[to_batches][lancedb.query.AsyncQueryBase.to_batches] and convert each batch to
pandas separately.
Examples
--------
>>> import asyncio
>>> from lancedb import connect_async
>>> async def doctest_example():
... conn = await connect_async("./.lancedb")
... table = await conn.create_table("my_table", data=[{"a": 1, "b": 2}])
... async for batch in await table.query().to_batches():
... batch_df = batch.to_pandas()
>>> asyncio.run(doctest_example())
Parameters
----------
flatten: Optional[Union[int, bool]]
If flatten is True, flatten all nested columns.
If flatten is an integer, flatten the nested columns up to the
specified depth.
If unspecified, do not flatten the nested columns.
timeout: Optional[timedelta]
The maximum time to wait for the query to complete.
If not specified, no timeout is applied. If the query does not
complete within the specified time, an error will be raised.
"""
return LOOP.run(self._inner.to_pandas(flatten, timeout))
def to_polars(
self,
timeout: Optional[timedelta] = None,
) -> "pl.DataFrame":
"""
Execute the query and collect the results into a Polars DataFrame.
This method will collect all results into memory before returning. If you
expect a large number of results, you may want to use
[to_batches][lancedb.query.AsyncQueryBase.to_batches] and convert each batch to
polars separately.
Parameters
----------
timeout: Optional[timedelta]
The maximum time to wait for the query to complete.
If not specified, no timeout is applied. If the query does not
complete within the specified time, an error will be raised.
Examples
--------
>>> import asyncio
>>> import polars as pl
>>> from lancedb import connect_async
>>> async def doctest_example():
... conn = await connect_async("./.lancedb")
... table = await conn.create_table("my_table", data=[{"a": 1, "b": 2}])
... async for batch in await table.query().to_batches():
... batch_df = pl.from_arrow(batch)
>>> asyncio.run(doctest_example())
"""
return LOOP.run(self._inner.to_polars(timeout))
def explain_plan(self, verbose: Optional[bool] = False):
"""Return the execution plan for this query.
Examples
--------
>>> import asyncio
>>> from lancedb import connect_async
>>> async def doctest_example():
... conn = await connect_async("./.lancedb")
... table = await conn.create_table("my_table", [{"vector": [99, 99]}])
... query = [100, 100]
... plan = await table.query().nearest_to([1, 2]).explain_plan(True)
... print(plan)
>>> asyncio.run(doctest_example()) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
ProjectionExec: expr=[vector@0 as vector, _distance@2 as _distance]
GlobalLimitExec: skip=0, fetch=10
FilterExec: _distance@2 IS NOT NULL
SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
KNNVectorDistance: metric=l2
LanceRead: uri=..., projection=[vector], ...
Parameters
----------
verbose : bool, default False
Use a verbose output format.
Returns
-------
plan : str
""" # noqa: E501
return LOOP.run(self._inner.explain_plan(verbose))
def analyze_plan(self):
"""Execute the query and display with runtime metrics.
Returns
-------
plan : str
"""
return LOOP.run(self._inner.analyze_plan())
class LanceTakeQueryBuilder(BaseQueryBuilder):
"""
Builder for parameterizing and executing take queries.
"""
def __init__(self, inner: AsyncTakeQuery):
super().__init__(inner)

View File

@@ -8,15 +8,7 @@ from typing import List, Optional
from lancedb import __version__
from .header import HeaderProvider
__all__ = [
"TimeoutConfig",
"RetryConfig",
"TlsConfig",
"ClientConfig",
"HeaderProvider",
]
__all__ = ["TimeoutConfig", "RetryConfig", "ClientConfig"]
@dataclass
@@ -120,43 +112,15 @@ class RetryConfig:
statuses: Optional[List[int]] = None
@dataclass
class TlsConfig:
"""TLS/mTLS configuration for the remote HTTP client.
Attributes
----------
cert_file: Optional[str]
Path to the client certificate file (PEM format) for mTLS authentication.
key_file: Optional[str]
Path to the client private key file (PEM format) for mTLS authentication.
ssl_ca_cert: Optional[str]
Path to the CA certificate file (PEM format) for server verification.
assert_hostname: bool
Whether to verify the hostname in the server's certificate. Default is True.
Set to False to disable hostname verification (use with caution).
"""
cert_file: Optional[str] = None
key_file: Optional[str] = None
ssl_ca_cert: Optional[str] = None
assert_hostname: bool = True
@dataclass
class ClientConfig:
user_agent: str = f"LanceDB-Python-Client/{__version__}"
retry_config: RetryConfig = field(default_factory=RetryConfig)
timeout_config: Optional[TimeoutConfig] = field(default_factory=TimeoutConfig)
extra_headers: Optional[dict] = None
id_delimiter: Optional[str] = None
tls_config: Optional[TlsConfig] = None
header_provider: Optional["HeaderProvider"] = None
def __post_init__(self):
if isinstance(self.retry_config, dict):
self.retry_config = RetryConfig(**self.retry_config)
if isinstance(self.timeout_config, dict):
self.timeout_config = TimeoutConfig(**self.timeout_config)
if isinstance(self.tls_config, dict):
self.tls_config = TlsConfig(**self.tls_config)

View File

@@ -96,73 +96,14 @@ class RemoteDBConnection(DBConnection):
def __repr__(self) -> str:
return f"RemoteConnect(name={self.db_name})"
@override
def list_namespaces(
self,
namespace: List[str] = [],
page_token: Optional[str] = None,
limit: int = 10,
) -> Iterable[str]:
"""List immediate child namespace names in the given namespace.
Parameters
----------
namespace: List[str], optional
The parent namespace to list namespaces in.
None or empty list represents root namespace.
page_token: str, optional
The token to use for pagination. If not present, start from the beginning.
limit: int, default 10
The size of the page to return.
Returns
-------
Iterable of str
List of immediate child namespace names
"""
return LOOP.run(
self._conn.list_namespaces(
namespace=namespace, page_token=page_token, limit=limit
)
)
@override
def create_namespace(self, namespace: List[str]) -> None:
"""Create a new namespace.
Parameters
----------
namespace: List[str]
The namespace identifier to create.
"""
LOOP.run(self._conn.create_namespace(namespace=namespace))
@override
def drop_namespace(self, namespace: List[str]) -> None:
"""Drop a namespace.
Parameters
----------
namespace: List[str]
The namespace identifier to drop.
"""
return LOOP.run(self._conn.drop_namespace(namespace=namespace))
@override
def table_names(
self,
page_token: Optional[str] = None,
limit: int = 10,
*,
namespace: List[str] = [],
self, page_token: Optional[str] = None, limit: int = 10
) -> Iterable[str]:
"""List the names of all tables in the database.
Parameters
----------
namespace: List[str], default []
The namespace to list tables in.
Empty list represents root namespace.
page_token: str
The last token to start the new page.
limit: int, default 10
@@ -172,18 +113,13 @@ class RemoteDBConnection(DBConnection):
-------
An iterator of table names.
"""
return LOOP.run(
self._conn.table_names(
namespace=namespace, start_after=page_token, limit=limit
)
)
return LOOP.run(self._conn.table_names(start_after=page_token, limit=limit))
@override
def open_table(
self,
name: str,
*,
namespace: List[str] = [],
storage_options: Optional[Dict[str, str]] = None,
index_cache_size: Optional[int] = None,
) -> Table:
@@ -193,9 +129,6 @@ class RemoteDBConnection(DBConnection):
----------
name: str
The name of the table.
namespace: List[str], optional
The namespace to open the table from.
None or empty list represents root namespace.
Returns
-------
@@ -209,54 +142,7 @@ class RemoteDBConnection(DBConnection):
" (there is no local cache to configure)"
)
table = LOOP.run(self._conn.open_table(name, namespace=namespace))
return RemoteTable(table, self.db_name)
def clone_table(
self,
target_table_name: str,
source_uri: str,
*,
target_namespace: List[str] = [],
source_version: Optional[int] = None,
source_tag: Optional[str] = None,
is_shallow: bool = True,
) -> Table:
"""Clone a table from a source table.
Parameters
----------
target_table_name: str
The name of the target table to create.
source_uri: str
The URI of the source table to clone from.
target_namespace: List[str], optional
The namespace for the target table.
None or empty list represents root namespace.
source_version: int, optional
The version of the source table to clone.
source_tag: str, optional
The tag of the source table to clone.
is_shallow: bool, default True
Whether to perform a shallow clone (True) or deep clone (False).
Currently only shallow clone is supported.
Returns
-------
A RemoteTable object representing the cloned table.
"""
from .table import RemoteTable
table = LOOP.run(
self._conn.clone_table(
target_table_name,
source_uri,
target_namespace=target_namespace,
source_version=source_version,
source_tag=source_tag,
is_shallow=is_shallow,
)
)
table = LOOP.run(self._conn.open_table(name))
return RemoteTable(table, self.db_name)
@override
@@ -269,8 +155,6 @@ class RemoteDBConnection(DBConnection):
fill_value: float = 0.0,
mode: Optional[str] = None,
embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None,
*,
namespace: List[str] = [],
) -> Table:
"""Create a [Table][lancedb.table.Table] in the database.
@@ -278,9 +162,6 @@ class RemoteDBConnection(DBConnection):
----------
name: str
The name of the table.
namespace: List[str], optional
The namespace to create the table in.
None or empty list represents root namespace.
data: The data to initialize the table, *optional*
User must provide at least one of `data` or `schema`.
Acceptable types are:
@@ -381,7 +262,6 @@ class RemoteDBConnection(DBConnection):
self._conn.create_table(
name,
data,
namespace=namespace,
mode=mode,
schema=schema,
on_bad_vectors=on_bad_vectors,
@@ -391,27 +271,18 @@ class RemoteDBConnection(DBConnection):
return RemoteTable(table, self.db_name)
@override
def drop_table(self, name: str, namespace: List[str] = []):
def drop_table(self, name: str):
"""Drop a table from the database.
Parameters
----------
name: str
The name of the table.
namespace: List[str], optional
The namespace to drop the table from.
None or empty list represents root namespace.
"""
LOOP.run(self._conn.drop_table(name, namespace=namespace))
LOOP.run(self._conn.drop_table(name))
@override
def rename_table(
self,
cur_name: str,
new_name: str,
cur_namespace: List[str] = [],
new_namespace: List[str] = [],
):
def rename_table(self, cur_name: str, new_name: str):
"""Rename a table in the database.
Parameters
@@ -421,14 +292,7 @@ class RemoteDBConnection(DBConnection):
new_name: str
The new name of the table.
"""
LOOP.run(
self._conn.rename_table(
cur_name,
new_name,
cur_namespace=cur_namespace,
new_namespace=new_namespace,
)
)
LOOP.run(self._conn.rename_table(cur_name, new_name))
async def close(self):
"""Close the connection to the database."""

View File

@@ -1,180 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""Header providers for LanceDB remote connections.
This module provides a flexible header management framework for LanceDB remote
connections, allowing users to implement custom header strategies for
authentication, request tracking, custom metadata, or any other header-based
requirements.
The module includes the HeaderProvider abstract base class and example implementations
(StaticHeaderProvider and OAuthProvider) that demonstrate common patterns.
The HeaderProvider interface is designed to be called before each request to the remote
server, enabling dynamic header scenarios where values may need to be
refreshed, rotated, or computed on-demand.
"""
from abc import ABC, abstractmethod
from typing import Dict, Optional, Callable, Any
import time
import threading
class HeaderProvider(ABC):
"""Abstract base class for providing custom headers for each request.
Users can implement this interface to provide dynamic headers for various purposes
such as authentication (OAuth tokens, API keys), request tracking (correlation IDs),
custom metadata, or any other header-based requirements. The provider is called
before each request to ensure fresh header values are always used.
Error Handling
--------------
If get_headers() raises an exception, the request will fail. Implementations
should handle recoverable errors internally (e.g., retry token refresh) and
only raise exceptions for unrecoverable errors.
"""
@abstractmethod
def get_headers(self) -> Dict[str, str]:
"""Get the latest headers to be added to requests.
This method is called before each request to the remote LanceDB server.
Implementations should return headers that will be merged with existing headers.
Returns
-------
Dict[str, str]
Dictionary of header names to values to add to the request.
Raises
------
Exception
If unable to fetch headers, the exception will be propagated
and the request will fail.
"""
pass
class StaticHeaderProvider(HeaderProvider):
"""Example implementation: A simple header provider that returns static headers.
This is an example implementation showing how to create a HeaderProvider
for cases where headers don't change during the session. Users can use this
as a reference for implementing their own providers.
Parameters
----------
headers : Dict[str, str]
Static headers to return for every request.
"""
def __init__(self, headers: Dict[str, str]):
"""Initialize with static headers.
Parameters
----------
headers : Dict[str, str]
Headers to return for every request.
"""
self._headers = headers.copy()
def get_headers(self) -> Dict[str, str]:
"""Return the static headers.
Returns
-------
Dict[str, str]
Copy of the static headers.
"""
return self._headers.copy()
class OAuthProvider(HeaderProvider):
"""Example implementation: OAuth token provider with automatic refresh.
This is an example implementation showing how to manage OAuth tokens
with automatic refresh when they expire. Users can use this as a reference
for implementing their own OAuth or token-based authentication providers.
Parameters
----------
token_fetcher : Callable[[], Dict[str, Any]]
Function that fetches a new token. Should return a dict with
'access_token' and optionally 'expires_in' (seconds until expiration).
refresh_buffer_seconds : int, optional
Number of seconds before expiration to trigger refresh. Default is 300
(5 minutes).
"""
def __init__(
self, token_fetcher: Callable[[], Any], refresh_buffer_seconds: int = 300
):
"""Initialize the OAuth provider.
Parameters
----------
token_fetcher : Callable[[], Any]
Function to fetch new tokens. Should return dict with
'access_token' and optionally 'expires_in'.
refresh_buffer_seconds : int, optional
Seconds before expiry to refresh token. Default 300.
"""
self._token_fetcher = token_fetcher
self._refresh_buffer = refresh_buffer_seconds
self._current_token: Optional[str] = None
self._token_expires_at: Optional[float] = None
self._refresh_lock = threading.Lock()
def _refresh_token_if_needed(self) -> None:
"""Refresh the token if it's expired or close to expiring."""
with self._refresh_lock:
# Check again inside the lock in case another thread refreshed
if self._needs_refresh():
token_data = self._token_fetcher()
self._current_token = token_data.get("access_token")
if not self._current_token:
raise ValueError("Token fetcher did not return 'access_token'")
# Set expiration if provided
expires_in = token_data.get("expires_in")
if expires_in:
self._token_expires_at = time.time() + expires_in
else:
# Token doesn't expire or expiration unknown
self._token_expires_at = None
def _needs_refresh(self) -> bool:
"""Check if token needs refresh."""
if self._current_token is None:
return True
if self._token_expires_at is None:
# No expiration info, assume token is valid
return False
# Refresh if we're within the buffer time of expiration
return time.time() >= (self._token_expires_at - self._refresh_buffer)
def get_headers(self) -> Dict[str, str]:
"""Get OAuth headers, refreshing token if needed.
Returns
-------
Dict[str, str]
Headers with Bearer token authorization.
Raises
------
Exception
If unable to fetch or refresh token.
"""
self._refresh_token_if_needed()
if not self._current_token:
raise RuntimeError("Failed to obtain OAuth token")
return {"Authorization": f"Bearer {self._current_token}"}

View File

@@ -26,7 +26,7 @@ from lancedb.common import DATA, VEC, VECTOR_COLUMN_NAME
from lancedb.merge import LanceMergeInsertBuilder
from lancedb.embeddings import EmbeddingFunctionRegistry
from ..query import LanceVectorQueryBuilder, LanceQueryBuilder, LanceTakeQueryBuilder
from ..query import LanceVectorQueryBuilder, LanceQueryBuilder
from ..table import AsyncTable, IndexStatistics, Query, Table, Tags
@@ -115,7 +115,6 @@ class RemoteTable(Table):
*,
replace: bool = False,
wait_timeout: timedelta = None,
name: Optional[str] = None,
):
"""Creates a scalar index
Parameters
@@ -140,11 +139,7 @@ class RemoteTable(Table):
LOOP.run(
self._table.create_index(
column,
config=config,
replace=replace,
wait_timeout=wait_timeout,
name=name,
column, config=config, replace=replace, wait_timeout=wait_timeout
)
)
@@ -166,7 +161,6 @@ class RemoteTable(Table):
ngram_min_length: int = 3,
ngram_max_length: int = 3,
prefix_only: bool = False,
name: Optional[str] = None,
):
config = FTS(
with_position=with_position,
@@ -183,11 +177,7 @@ class RemoteTable(Table):
)
LOOP.run(
self._table.create_index(
column,
config=config,
replace=replace,
wait_timeout=wait_timeout,
name=name,
column, config=config, replace=replace, wait_timeout=wait_timeout
)
)
@@ -204,8 +194,6 @@ class RemoteTable(Table):
wait_timeout: Optional[timedelta] = None,
*,
num_bits: int = 8,
name: Optional[str] = None,
train: bool = True,
):
"""Create an index on the table.
Currently, the only parameters that matter are
@@ -282,11 +270,7 @@ class RemoteTable(Table):
LOOP.run(
self._table.create_index(
vector_column_name,
config=config,
wait_timeout=wait_timeout,
name=name,
train=train,
vector_column_name, config=config, wait_timeout=wait_timeout
)
)
@@ -633,12 +617,6 @@ class RemoteTable(Table):
def stats(self):
return LOOP.run(self._table.stats())
def take_offsets(self, offsets: list[int]) -> LanceTakeQueryBuilder:
return LanceTakeQueryBuilder(self._table.take_offsets(offsets))
def take_row_ids(self, row_ids: list[int]) -> LanceTakeQueryBuilder:
return LanceTakeQueryBuilder(self._table.take_row_ids(row_ids))
def uses_v2_manifest_paths(self) -> bool:
raise NotImplementedError(
"uses_v2_manifest_paths() is not supported on the LanceDB Cloud"

View File

@@ -9,7 +9,6 @@ from .linear_combination import LinearCombinationReranker
from .openai import OpenaiReranker
from .jinaai import JinaReranker
from .rrf import RRFReranker
from .mrr import MRRReranker
from .answerdotai import AnswerdotaiRerankers
from .voyageai import VoyageAIReranker
@@ -24,5 +23,4 @@ __all__ = [
"RRFReranker",
"AnswerdotaiRerankers",
"VoyageAIReranker",
"MRRReranker",
]

View File

@@ -1,169 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
from typing import Union, List, TYPE_CHECKING
import pyarrow as pa
import numpy as np
from collections import defaultdict
from .base import Reranker
if TYPE_CHECKING:
from ..table import LanceVectorQueryBuilder
class MRRReranker(Reranker):
"""
Reranks the results using Mean Reciprocal Rank (MRR) algorithm based
on the scores of vector and FTS search.
Algorithm reference - https://en.wikipedia.org/wiki/Mean_reciprocal_rank
MRR calculates the average of reciprocal ranks across different search results.
For each document, it computes the reciprocal of its rank in each system,
then takes the mean of these reciprocal ranks as the final score.
Parameters
----------
weight_vector : float, default 0.5
Weight for vector search results (0.0 to 1.0)
weight_fts : float, default 0.5
Weight for FTS search results (0.0 to 1.0)
Note: weight_vector + weight_fts should equal 1.0
return_score : str, default "relevance"
Options are "relevance" or "all"
The type of score to return. If "relevance", will return only the relevance
score. If "all", will return all scores from the vector and FTS search along
with the relevance score.
"""
def __init__(
self,
weight_vector: float = 0.5,
weight_fts: float = 0.5,
return_score="relevance",
):
if not (0.0 <= weight_vector <= 1.0):
raise ValueError("weight_vector must be between 0.0 and 1.0")
if not (0.0 <= weight_fts <= 1.0):
raise ValueError("weight_fts must be between 0.0 and 1.0")
if abs(weight_vector + weight_fts - 1.0) > 1e-6:
raise ValueError("weight_vector + weight_fts must equal 1.0")
super().__init__(return_score)
self.weight_vector = weight_vector
self.weight_fts = weight_fts
def rerank_hybrid(
self,
query: str, # noqa: F821
vector_results: pa.Table,
fts_results: pa.Table,
):
vector_ids = vector_results["_rowid"].to_pylist() if vector_results else []
fts_ids = fts_results["_rowid"].to_pylist() if fts_results else []
# Maps result_id to list of (type, reciprocal_rank)
mrr_score_map = defaultdict(list)
if vector_ids:
for rank, result_id in enumerate(vector_ids, 1):
reciprocal_rank = 1.0 / rank
mrr_score_map[result_id].append(("vector", reciprocal_rank))
if fts_ids:
for rank, result_id in enumerate(fts_ids, 1):
reciprocal_rank = 1.0 / rank
mrr_score_map[result_id].append(("fts", reciprocal_rank))
final_mrr_scores = {}
for result_id, scores in mrr_score_map.items():
vector_rr = 0.0
fts_rr = 0.0
for score_type, reciprocal_rank in scores:
if score_type == "vector":
vector_rr = reciprocal_rank
elif score_type == "fts":
fts_rr = reciprocal_rank
# If a document doesn't appear, its reciprocal rank is 0
weighted_mrr = self.weight_vector * vector_rr + self.weight_fts * fts_rr
final_mrr_scores[result_id] = weighted_mrr
combined_results = self.merge_results(vector_results, fts_results)
combined_row_ids = combined_results["_rowid"].to_pylist()
relevance_scores = [final_mrr_scores[row_id] for row_id in combined_row_ids]
combined_results = combined_results.append_column(
"_relevance_score", pa.array(relevance_scores, type=pa.float32())
)
combined_results = combined_results.sort_by(
[("_relevance_score", "descending")]
)
if self.score == "relevance":
combined_results = self._keep_relevance_score(combined_results)
return combined_results
def rerank_multivector(
self,
vector_results: Union[List[pa.Table], List["LanceVectorQueryBuilder"]],
query: str = None,
deduplicate: bool = True, # noqa: F821
):
"""
Reranks the results from multiple vector searches using MRR algorithm.
Each vector search result is treated as a separate ranking system,
and MRR calculates the mean of reciprocal ranks across all systems.
This cannot reuse rerank_hybrid because MRR semantics require treating
each vector result as a separate ranking system.
"""
if not all(isinstance(v, type(vector_results[0])) for v in vector_results):
raise ValueError(
"All elements in vector_results should be of the same type"
)
# avoid circular import
if type(vector_results[0]).__name__ == "LanceVectorQueryBuilder":
vector_results = [result.to_arrow() for result in vector_results]
elif not isinstance(vector_results[0], pa.Table):
raise ValueError(
"vector_results should be a list of pa.Table or LanceVectorQueryBuilder"
)
if not all("_rowid" in result.column_names for result in vector_results):
raise ValueError(
"'_rowid' is required for deduplication. \
add _rowid to search results like this: \
`search().with_row_id(True)`"
)
mrr_score_map = defaultdict(list)
for result_table in vector_results:
result_ids = result_table["_rowid"].to_pylist()
for rank, result_id in enumerate(result_ids, 1):
reciprocal_rank = 1.0 / rank
mrr_score_map[result_id].append(reciprocal_rank)
final_mrr_scores = {}
for result_id, reciprocal_ranks in mrr_score_map.items():
mean_rr = np.mean(reciprocal_ranks)
final_mrr_scores[result_id] = mean_rr
combined = pa.concat_tables(vector_results, **self._concat_tables_args)
combined = self._deduplicate(combined)
combined_row_ids = combined["_rowid"].to_pylist()
relevance_scores = [final_mrr_scores[row_id] for row_id in combined_row_ids]
combined = combined.append_column(
"_relevance_score", pa.array(relevance_scores, type=pa.float32())
)
combined = combined.sort_by([("_relevance_score", "descending")])
if self.score == "relevance":
combined = self._keep_relevance_score(combined)
return combined

View File

@@ -51,7 +51,6 @@ from .query import (
AsyncFTSQuery,
AsyncHybridQuery,
AsyncQuery,
AsyncTakeQuery,
AsyncVectorQuery,
FullTextQuery,
LanceEmptyQueryBuilder,
@@ -59,7 +58,6 @@ from .query import (
LanceHybridQueryBuilder,
LanceQueryBuilder,
LanceVectorQueryBuilder,
LanceTakeQueryBuilder,
Query,
)
from .util import (
@@ -689,9 +687,6 @@ class Table(ABC):
sample_rate: int = 256,
m: int = 20,
ef_construction: int = 300,
name: Optional[str] = None,
train: bool = True,
target_partition_size: Optional[int] = None,
):
"""Create an index on the table.
@@ -724,11 +719,6 @@ class Table(ABC):
Only 4 and 8 are supported.
wait_timeout: timedelta, optional
The timeout to wait if indexing is asynchronous.
name: str, optional
The name of the index. If not provided, a default name will be generated.
train: bool, default True
Whether to train the index with existing data. Vector indices always train
with existing data.
"""
raise NotImplementedError
@@ -784,7 +774,6 @@ class Table(ABC):
replace: bool = True,
index_type: ScalarIndexType = "BTREE",
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
):
"""Create a scalar index on a column.
@@ -799,8 +788,6 @@ class Table(ABC):
The type of index to create.
wait_timeout: timedelta, optional
The timeout to wait if indexing is asynchronous.
name: str, optional
The name of the index. If not provided, a default name will be generated.
Examples
--------
@@ -863,7 +850,6 @@ class Table(ABC):
ngram_max_length: int = 3,
prefix_only: bool = False,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
):
"""Create a full-text search index on the table.
@@ -928,8 +914,6 @@ class Table(ABC):
Whether to only index the prefix of the token for ngram tokenizer.
wait_timeout: timedelta, optional
The timeout to wait if indexing is asynchronous.
name: str, optional
The name of the index. If not provided, a default name will be generated.
"""
raise NotImplementedError
@@ -1119,120 +1103,6 @@ class Table(ABC):
"""
raise NotImplementedError
@abstractmethod
def take_offsets(
self, offsets: list[int], *, with_row_id: bool = False
) -> LanceTakeQueryBuilder:
"""
Take a list of offsets from the table.
Offsets are 0-indexed and relative to the current version of the table. Offsets
are not stable. A row with an offset of N may have a different offset in a
different version of the table (e.g. if an earlier row is deleted).
Offsets are mostly useful for sampling as the set of all valid offsets is easily
known in advance to be [0, len(table)).
No guarantees are made regarding the order in which results are returned. If
you desire an output order that matches the order of the given offsets, you will
need to add the row offset column to the output and align it yourself.
Parameters
----------
offsets: list[int]
The offsets to take.
Returns
-------
pa.RecordBatch
A record batch containing the rows at the given offsets.
"""
def __getitems__(self, offsets: list[int]) -> pa.RecordBatch:
"""
Take a list of offsets from the table and return as a record batch.
This method uses the `take_offsets` method to take the rows. However, it
aligns the offsets to the passed in offsets. This means the return type
is a record batch (and so users should take care not to pass in too many
offsets)
Note: this method is primarily intended to fulfill the Dataset contract
for pytorch.
Parameters
----------
offsets: list[int]
The offsets to take.
Returns
-------
pa.RecordBatch
A record batch containing the rows at the given offsets.
"""
# We don't know the order of the results at all. So we calculate a permutation
# for ordering the given offsets. Then we load the data with the _rowoffset
# column. Then we sort by _rowoffset and apply the inverse of the permutation
# that we calculated.
#
# Note: this is potentially a lot of memory copy if we're operating on large
# batches :(
num_offsets = len(offsets)
indices = list(range(num_offsets))
permutation = sorted(indices, key=lambda idx: offsets[idx])
permutation_inv = [0] * num_offsets
for i in range(num_offsets):
permutation_inv[permutation[i]] = i
columns = self.schema.names
columns.append("_rowoffset")
tbl = (
self.take_offsets(offsets)
.select(columns)
.to_arrow()
.sort_by("_rowoffset")
.take(permutation_inv)
.combine_chunks()
.drop_columns(["_rowoffset"])
)
return tbl
@abstractmethod
def take_row_ids(
self, row_ids: list[int], *, with_row_id: bool = False
) -> LanceTakeQueryBuilder:
"""
Take a list of row ids from the table.
Row ids are not stable and are relative to the current version of the table.
They can change due to compaction and updates.
No guarantees are made regarding the order in which results are returned. If
you desire an output order that matches the order of the given ids, you will
need to add the row id column to the output and align it yourself.
Unlike offsets, row ids are not 0-indexed and no assumptions should be made
about the possible range of row ids. In order to use this method you must
first obtain the row ids by scanning or searching the table.
Even so, row ids are more stable than offsets and can be useful in some
situations.
There is an ongoing effort to make row ids stable which is tracked at
https://github.com/lancedb/lancedb/issues/1120
Parameters
----------
row_ids: list[int]
The row ids to take.
Returns
-------
AsyncTakeQuery
A query object that can be executed to get the rows.
"""
@abstractmethod
def _execute_query(
self,
@@ -1470,7 +1340,10 @@ class Table(ABC):
be deleted unless they are at least 7 days old. If delete_unverified is True
then these files will be deleted regardless of their age.
retrain: bool, default False
This parameter is no longer used and is deprecated.
If True, retrain the vector indices, this would refine the IVF clustering
and quantization, which may improve the search accuracy. It's faster than
re-creating the index from scratch, so it's recommended to try this first,
when the data distribution has changed significantly.
Experimental API
----------------
@@ -1704,16 +1577,13 @@ class LanceTable(Table):
connection: "LanceDBConnection",
name: str,
*,
namespace: List[str] = [],
storage_options: Optional[Dict[str, str]] = None,
index_cache_size: Optional[int] = None,
):
self._conn = connection
self._namespace = namespace
self._table = LOOP.run(
connection._conn.open_table(
name,
namespace=namespace,
storage_options=storage_options,
index_cache_size=index_cache_size,
)
@@ -1724,8 +1594,8 @@ class LanceTable(Table):
return self._table.name
@classmethod
def open(cls, db, name, *, namespace: List[str] = [], **kwargs):
tbl = cls(db, name, namespace=namespace, **kwargs)
def open(cls, db, name, **kwargs):
tbl = cls(db, name, **kwargs)
# check the dataset exists
try:
@@ -1778,12 +1648,6 @@ class LanceTable(Table):
"""Get the current version of the table"""
return LOOP.run(self._table.version())
def take_offsets(self, offsets: list[int]) -> LanceTakeQueryBuilder:
return LanceTakeQueryBuilder(self._table.take_offsets(offsets))
def take_row_ids(self, row_ids: list[int]) -> LanceTakeQueryBuilder:
return LanceTakeQueryBuilder(self._table.take_row_ids(row_ids))
@property
def tags(self) -> Tags:
"""Tag management for the table.
@@ -1997,10 +1861,6 @@ class LanceTable(Table):
sample_rate: int = 256,
m: int = 20,
ef_construction: int = 300,
*,
name: Optional[str] = None,
train: bool = True,
target_partition_size: Optional[int] = None,
):
"""Create an index on the table."""
if accelerator is not None:
@@ -2017,7 +1877,6 @@ class LanceTable(Table):
num_bits=num_bits,
m=m,
ef_construction=ef_construction,
target_partition_size=target_partition_size,
)
self.checkout_latest()
return
@@ -2027,7 +1886,6 @@ class LanceTable(Table):
num_partitions=num_partitions,
max_iterations=max_iterations,
sample_rate=sample_rate,
target_partition_size=target_partition_size,
)
elif index_type == "IVF_PQ":
config = IvfPq(
@@ -2037,7 +1895,6 @@ class LanceTable(Table):
num_bits=num_bits,
max_iterations=max_iterations,
sample_rate=sample_rate,
target_partition_size=target_partition_size,
)
elif index_type == "IVF_HNSW_PQ":
config = HnswPq(
@@ -2049,7 +1906,6 @@ class LanceTable(Table):
sample_rate=sample_rate,
m=m,
ef_construction=ef_construction,
target_partition_size=target_partition_size,
)
elif index_type == "IVF_HNSW_SQ":
config = HnswSq(
@@ -2059,7 +1915,6 @@ class LanceTable(Table):
sample_rate=sample_rate,
m=m,
ef_construction=ef_construction,
target_partition_size=target_partition_size,
)
else:
raise ValueError(f"Unknown index type {index_type}")
@@ -2069,8 +1924,6 @@ class LanceTable(Table):
vector_column_name,
replace=replace,
config=config,
name=name,
train=train,
)
)
@@ -2115,7 +1968,6 @@ class LanceTable(Table):
*,
replace: bool = True,
index_type: ScalarIndexType = "BTREE",
name: Optional[str] = None,
):
if index_type == "BTREE":
config = BTree()
@@ -2126,7 +1978,7 @@ class LanceTable(Table):
else:
raise ValueError(f"Unknown index type {index_type}")
return LOOP.run(
self._table.create_index(column, replace=replace, config=config, name=name)
self._table.create_index(column, replace=replace, config=config)
)
def create_fts_index(
@@ -2150,7 +2002,6 @@ class LanceTable(Table):
ngram_min_length: int = 3,
ngram_max_length: int = 3,
prefix_only: bool = False,
name: Optional[str] = None,
):
if not use_tantivy:
if not isinstance(field_names, str):
@@ -2188,7 +2039,6 @@ class LanceTable(Table):
field_names,
replace=replace,
config=config,
name=name,
)
)
return
@@ -2555,7 +2405,6 @@ class LanceTable(Table):
fill_value: float = 0.0,
embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None,
*,
namespace: List[str] = [],
storage_options: Optional[Dict[str, str | bool]] = None,
data_storage_version: Optional[str] = None,
enable_v2_manifest_paths: Optional[bool] = None,
@@ -2615,7 +2464,6 @@ class LanceTable(Table):
"""
self = cls.__new__(cls)
self._conn = db
self._namespace = namespace
if data_storage_version is not None:
warnings.warn(
@@ -2648,7 +2496,6 @@ class LanceTable(Table):
on_bad_vectors=on_bad_vectors,
fill_value=fill_value,
embedding_functions=embedding_functions,
namespace=namespace,
storage_options=storage_options,
)
)
@@ -2832,7 +2679,10 @@ class LanceTable(Table):
be deleted unless they are at least 7 days old. If delete_unverified is True
then these files will be deleted regardless of their age.
retrain: bool, default False
This parameter is no longer used and is deprecated.
If True, retrain the vector indices, this would refine the IVF clustering
and quantization, which may improve the search accuracy. It's faster than
re-creating the index from scratch, so it's recommended to try this first,
when the data distribution has changed significantly.
Experimental API
----------------
@@ -3333,8 +3183,6 @@ class AsyncTable:
Union[IvfFlat, IvfPq, HnswPq, HnswSq, BTree, Bitmap, LabelList, FTS]
] = None,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
train: bool = True,
):
"""Create an index to speed up queries
@@ -3361,11 +3209,6 @@ class AsyncTable:
creating an index object.
wait_timeout: timedelta, optional
The timeout to wait if indexing is asynchronous.
name: str, optional
The name of the index. If not provided, a default name will be generated.
train: bool, default True
Whether to train the index with existing data. Vector indices always train
with existing data.
"""
if config is not None:
if not isinstance(
@@ -3377,12 +3220,7 @@ class AsyncTable:
)
try:
await self._inner.create_index(
column,
index=config,
replace=replace,
wait_timeout=wait_timeout,
name=name,
train=train,
column, index=config, replace=replace, wait_timeout=wait_timeout
)
except ValueError as e:
if "not support the requested language" in str(e):
@@ -4192,58 +4030,6 @@ class AsyncTable:
"""
await self._inner.restore(version)
def take_offsets(self, offsets: list[int]) -> AsyncTakeQuery:
"""
Take a list of offsets from the table.
Offsets are 0-indexed and relative to the current version of the table. Offsets
are not stable. A row with an offset of N may have a different offset in a
different version of the table (e.g. if an earlier row is deleted).
Offsets are mostly useful for sampling as the set of all valid offsets is easily
known in advance to be [0, len(table)).
Parameters
----------
offsets: list[int]
The offsets to take.
Returns
-------
pa.RecordBatch
A record batch containing the rows at the given offsets.
"""
return AsyncTakeQuery(self._inner.take_offsets(offsets))
def take_row_ids(self, row_ids: list[int]) -> AsyncTakeQuery:
"""
Take a list of row ids from the table.
Row ids are not stable and are relative to the current version of the table.
They can change due to compaction and updates.
Unlike offsets, row ids are not 0-indexed and no assumptions should be made
about the possible range of row ids. In order to use this method you must
first obtain the row ids by scanning or searching the table.
Even so, row ids are more stable than offsets and can be useful in some
situations.
There is an ongoing effort to make row ids stable which is tracked at
https://github.com/lancedb/lancedb/issues/1120
Parameters
----------
row_ids: list[int]
The row ids to take.
Returns
-------
AsyncTakeQuery
A query object that can be executed to get the rows.
"""
return AsyncTakeQuery(self._inner.take_row_ids(row_ids))
@property
def tags(self) -> AsyncTags:
"""Tag management for the dataset.
@@ -4292,7 +4078,10 @@ class AsyncTable:
be deleted unless they are at least 7 days old. If delete_unverified is True
then these files will be deleted regardless of their age.
retrain: bool, default False
This parameter is no longer used and is deprecated.
If True, retrain the vector indices, this would refine the IVF clustering
and quantization, which may improve the search accuracy. It's faster than
re-creating the index from scratch, so it's recommended to try this first,
when the data distribution has changed significantly.
Experimental API
----------------
@@ -4315,19 +4104,10 @@ class AsyncTable:
cleanup_since_ms: Optional[int] = None
if cleanup_older_than is not None:
cleanup_since_ms = round(cleanup_older_than.total_seconds() * 1000)
if retrain:
import warnings
warnings.warn(
"The 'retrain' parameter is deprecated and will be removed in a "
"future version.",
DeprecationWarning,
)
return await self._inner.optimize(
cleanup_since_ms=cleanup_since_ms,
delete_unverified=delete_unverified,
retrain=retrain,
)
async def list_indices(self) -> Iterable[IndexConfig]:

View File

@@ -175,18 +175,6 @@ def test_table_names(tmp_db: lancedb.DBConnection):
tmp_db.create_table("test3", data=data)
assert tmp_db.table_names() == ["test1", "test2", "test3"]
# Test that positional arguments for page_token and limit
result = list(tmp_db.table_names("test1", 1)) # page_token="test1", limit=1
assert result == ["test2"], f"Expected ['test2'], got {result}"
# Test mixed positional and keyword arguments
result = list(tmp_db.table_names("test2", limit=2))
assert result == ["test3"], f"Expected ['test3'], got {result}"
# Test that namespace parameter can be passed as keyword
result = list(tmp_db.table_names(namespace=[]))
assert len(result) == 3
@pytest.mark.asyncio
async def test_table_names_async(tmp_path):
@@ -740,210 +728,3 @@ def test_bypass_vector_index_sync(tmp_db: lancedb.DBConnection):
table.search(sample_key).bypass_vector_index().explain_plan(verbose=True)
)
assert "KNN" in plan_without_index
def test_local_namespace_operations(tmp_path):
"""Test that local mode namespace operations behave as expected."""
# Create a local database connection
db = lancedb.connect(tmp_path)
# Test list_namespaces returns empty list for root namespace
namespaces = list(db.list_namespaces())
assert namespaces == []
# Test list_namespaces with non-empty namespace raises NotImplementedError
with pytest.raises(
NotImplementedError,
match="Namespace operations are not supported for listing database",
):
list(db.list_namespaces(namespace=["test"]))
def test_local_create_namespace_not_supported(tmp_path):
"""Test that create_namespace is not supported in local mode."""
db = lancedb.connect(tmp_path)
with pytest.raises(
NotImplementedError,
match="Namespace operations are not supported for listing database",
):
db.create_namespace(["test_namespace"])
def test_local_drop_namespace_not_supported(tmp_path):
"""Test that drop_namespace is not supported in local mode."""
db = lancedb.connect(tmp_path)
with pytest.raises(
NotImplementedError,
match="Namespace operations are not supported for listing database",
):
db.drop_namespace(["test_namespace"])
def test_local_table_operations_with_namespace_raise_error(tmp_path):
"""
Test that table operations with namespace parameter
raise ValueError in local mode.
"""
db = lancedb.connect(tmp_path)
# Create some test data
data = [{"vector": [1.0, 2.0], "item": "test"}]
schema = pa.schema(
[pa.field("vector", pa.list_(pa.float32(), 2)), pa.field("item", pa.string())]
)
# Test create_table with namespace - should raise ValueError
with pytest.raises(
NotImplementedError,
match="Namespace parameter is not supported for listing database",
):
db.create_table(
"test_table_with_ns", data=data, schema=schema, namespace=["test_ns"]
)
# Create table normally for other tests
db.create_table("test_table", data=data, schema=schema)
assert "test_table" in db.table_names()
# Test open_table with namespace - should raise ValueError
with pytest.raises(
NotImplementedError,
match="Namespace parameter is not supported for listing database",
):
db.open_table("test_table", namespace=["test_ns"])
# Test table_names with namespace - should raise ValueError
with pytest.raises(
NotImplementedError,
match="Namespace parameter is not supported for listing database",
):
list(db.table_names(namespace=["test_ns"]))
# Test drop_table with namespace - should raise ValueError
with pytest.raises(
NotImplementedError,
match="Namespace parameter is not supported for listing database",
):
db.drop_table("test_table", namespace=["test_ns"])
# Test table_names without namespace - should work normally
tables_root = list(db.table_names())
assert "test_table" in tables_root
def test_clone_table_latest_version(tmp_path):
"""Test cloning a table with the latest version (default behavior)"""
import os
db = lancedb.connect(tmp_path)
# Create source table with some data
data = [
{"id": 1, "text": "hello", "vector": [1.0, 2.0]},
{"id": 2, "text": "world", "vector": [3.0, 4.0]},
]
source_table = db.create_table("source", data=data)
# Add more data to create a new version
more_data = [{"id": 3, "text": "test", "vector": [5.0, 6.0]}]
source_table.add(more_data)
# Clone the table (should get latest version with 3 rows)
source_uri = os.path.join(tmp_path, "source.lance")
cloned_table = db.clone_table("cloned", source_uri)
# Verify cloned table has all 3 rows
assert cloned_table.count_rows() == 3
assert "cloned" in db.table_names()
# Verify data matches
cloned_data = cloned_table.to_pandas()
assert len(cloned_data) == 3
assert set(cloned_data["id"].tolist()) == {1, 2, 3}
def test_clone_table_specific_version(tmp_path):
"""Test cloning a table from a specific version"""
import os
db = lancedb.connect(tmp_path)
# Create source table with initial data
data = [
{"id": 1, "text": "hello", "vector": [1.0, 2.0]},
{"id": 2, "text": "world", "vector": [3.0, 4.0]},
]
source_table = db.create_table("source", data=data)
# Get the initial version
initial_version = source_table.version
# Add more data to create a new version
more_data = [{"id": 3, "text": "test", "vector": [5.0, 6.0]}]
source_table.add(more_data)
# Verify source now has 3 rows
assert source_table.count_rows() == 3
# Clone from the initial version (should have only 2 rows)
source_uri = os.path.join(tmp_path, "source.lance")
cloned_table = db.clone_table("cloned", source_uri, source_version=initial_version)
# Verify cloned table has only the initial 2 rows
assert cloned_table.count_rows() == 2
cloned_data = cloned_table.to_pandas()
assert set(cloned_data["id"].tolist()) == {1, 2}
def test_clone_table_with_tag(tmp_path):
"""Test cloning a table from a tagged version"""
import os
db = lancedb.connect(tmp_path)
# Create source table with initial data
data = [
{"id": 1, "text": "hello", "vector": [1.0, 2.0]},
{"id": 2, "text": "world", "vector": [3.0, 4.0]},
]
source_table = db.create_table("source", data=data)
# Create a tag for the current version
source_table.tags.create("v1.0", source_table.version)
# Add more data after the tag
more_data = [{"id": 3, "text": "test", "vector": [5.0, 6.0]}]
source_table.add(more_data)
# Verify source now has 3 rows
assert source_table.count_rows() == 3
# Clone from the tagged version (should have only 2 rows)
source_uri = os.path.join(tmp_path, "source.lance")
cloned_table = db.clone_table("cloned", source_uri, source_tag="v1.0")
# Verify cloned table has only the tagged version's 2 rows
assert cloned_table.count_rows() == 2
cloned_data = cloned_table.to_pandas()
assert set(cloned_data["id"].tolist()) == {1, 2}
def test_clone_table_deep_clone_fails(tmp_path):
"""Test that deep clone raises an unsupported error"""
import os
db = lancedb.connect(tmp_path)
# Create source table with some data
data = [
{"id": 1, "text": "hello", "vector": [1.0, 2.0]},
{"id": 2, "text": "world", "vector": [3.0, 4.0]},
]
db.create_table("source", data=data)
# Try to create a deep clone (should fail)
source_uri = os.path.join(tmp_path, "source.lance")
with pytest.raises(Exception, match="Deep clone is not yet implemented"):
db.clone_table("cloned", source_uri, is_shallow=False)

View File

@@ -114,63 +114,6 @@ def test_embedding_function_variables():
assert func.safe_model_dump()["secret_key"] == "$var:secret"
def test_parse_functions_with_variables():
@register("variable-parsing-test")
class VariableParsingFunction(TextEmbeddingFunction):
api_key: str
base_url: Optional[str] = None
@staticmethod
def sensitive_keys():
return ["api_key"]
def ndims(self):
return 10
def generate_embeddings(self, texts):
# Mock implementation that just returns random embeddings
# In real usage, this would use the api_key to call an API
return [np.random.rand(self.ndims()).tolist() for _ in texts]
registry = EmbeddingFunctionRegistry.get_instance()
registry.set_var("test_api_key", "sk-test-key-12345")
registry.set_var("test_base_url", "https://api.example.com")
conf = EmbeddingFunctionConfig(
source_column="text",
vector_column="vector",
function=registry.get("variable-parsing-test").create(
api_key="$var:test_api_key", base_url="$var:test_base_url"
),
)
metadata = registry.get_table_metadata([conf])
# Create a mock arrow table with the metadata
schema = pa.schema(
[pa.field("text", pa.string()), pa.field("vector", pa.list_(pa.float32(), 10))]
)
table = pa.table({"text": [], "vector": []}, schema=schema)
table = table.replace_schema_metadata(metadata)
ds = lance.write_dataset(table, "memory://")
configs = registry.parse_functions(ds.schema.metadata)
assert "vector" in configs
parsed_func = configs["vector"].function
assert parsed_func.api_key == "sk-test-key-12345"
assert parsed_func.base_url == "https://api.example.com"
embeddings = parsed_func.generate_embeddings(["test text"])
assert len(embeddings) == 1
assert len(embeddings[0]) == 10
assert parsed_func.safe_model_dump()["api_key"] == "$var:test_api_key"
def test_embedding_with_bad_results(tmp_path):
@register("null-embedding")
class NullEmbeddingFunction(TextEmbeddingFunction):

View File

@@ -157,16 +157,7 @@ def test_create_index_with_stemming(tmp_path, table):
def test_create_inverted_index(table, use_tantivy, with_position):
if use_tantivy and not with_position:
pytest.skip("we don't support building a tantivy index without position")
table.create_fts_index(
"text",
use_tantivy=use_tantivy,
with_position=with_position,
name="custom_fts_index",
)
if not use_tantivy:
indices = table.list_indices()
fts_indices = [i for i in indices if i.index_type == "FTS"]
assert any(i.name == "custom_fts_index" for i in fts_indices)
table.create_fts_index("text", use_tantivy=use_tantivy, with_position=with_position)
def test_populate_index(tmp_path, table):

View File

@@ -1,237 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import concurrent.futures
import pytest
import time
import threading
from typing import Dict
from lancedb.remote import ClientConfig, HeaderProvider
from lancedb.remote.header import StaticHeaderProvider, OAuthProvider
class TestStaticHeaderProvider:
def test_init(self):
"""Test StaticHeaderProvider initialization."""
headers = {"X-API-Key": "test-key", "X-Custom": "value"}
provider = StaticHeaderProvider(headers)
assert provider._headers == headers
def test_get_headers(self):
"""Test get_headers returns correct headers."""
headers = {"X-API-Key": "test-key", "X-Custom": "value"}
provider = StaticHeaderProvider(headers)
result = provider.get_headers()
assert result == headers
# Ensure it returns a copy
result["X-Modified"] = "modified"
result2 = provider.get_headers()
assert "X-Modified" not in result2
class TestOAuthProvider:
def test_init(self):
"""Test OAuthProvider initialization."""
def fetcher():
return {"access_token": "token123", "expires_in": 3600}
provider = OAuthProvider(fetcher)
assert provider._token_fetcher is fetcher
assert provider._refresh_buffer == 300
assert provider._current_token is None
assert provider._token_expires_at is None
def test_get_headers_first_time(self):
"""Test get_headers fetches token on first call."""
def fetcher():
return {"access_token": "token123", "expires_in": 3600}
provider = OAuthProvider(fetcher)
headers = provider.get_headers()
assert headers == {"Authorization": "Bearer token123"}
assert provider._current_token == "token123"
assert provider._token_expires_at is not None
def test_token_refresh(self):
"""Test token refresh when expired."""
call_count = 0
tokens = ["token1", "token2"]
def fetcher():
nonlocal call_count
token = tokens[call_count]
call_count += 1
return {"access_token": token, "expires_in": 1} # Expires in 1 second
provider = OAuthProvider(fetcher, refresh_buffer_seconds=0)
# First call
headers1 = provider.get_headers()
assert headers1 == {"Authorization": "Bearer token1"}
# Wait for token to expire
time.sleep(1.1)
# Second call should refresh
headers2 = provider.get_headers()
assert headers2 == {"Authorization": "Bearer token2"}
assert call_count == 2
def test_no_expiry_info(self):
"""Test handling tokens without expiry information."""
def fetcher():
return {"access_token": "permanent_token"}
provider = OAuthProvider(fetcher)
headers = provider.get_headers()
assert headers == {"Authorization": "Bearer permanent_token"}
assert provider._token_expires_at is None
# Should not refresh on second call
headers2 = provider.get_headers()
assert headers2 == {"Authorization": "Bearer permanent_token"}
def test_missing_access_token(self):
"""Test error handling when access_token is missing."""
def fetcher():
return {"expires_in": 3600} # Missing access_token
provider = OAuthProvider(fetcher)
with pytest.raises(
ValueError, match="Token fetcher did not return 'access_token'"
):
provider.get_headers()
def test_sync_method(self):
"""Test synchronous get_headers method."""
def fetcher():
return {"access_token": "sync_token", "expires_in": 3600}
provider = OAuthProvider(fetcher)
headers = provider.get_headers()
assert headers == {"Authorization": "Bearer sync_token"}
class TestClientConfigIntegration:
def test_client_config_with_header_provider(self):
"""Test ClientConfig can accept a HeaderProvider."""
provider = StaticHeaderProvider({"X-Test": "value"})
config = ClientConfig(header_provider=provider)
assert config.header_provider is provider
def test_client_config_without_header_provider(self):
"""Test ClientConfig works without HeaderProvider."""
config = ClientConfig()
assert config.header_provider is None
class CustomProvider(HeaderProvider):
"""Custom provider for testing abstract class."""
def get_headers(self) -> Dict[str, str]:
return {"X-Custom": "custom-value"}
class TestCustomHeaderProvider:
def test_custom_provider(self):
"""Test custom HeaderProvider implementation."""
provider = CustomProvider()
headers = provider.get_headers()
assert headers == {"X-Custom": "custom-value"}
class ErrorProvider(HeaderProvider):
"""Provider that raises errors for testing error handling."""
def __init__(self, error_message: str = "Test error"):
self.error_message = error_message
self.call_count = 0
def get_headers(self) -> Dict[str, str]:
self.call_count += 1
raise RuntimeError(self.error_message)
class TestErrorHandling:
def test_provider_error_propagation(self):
"""Test that errors from header provider are properly propagated."""
provider = ErrorProvider("Authentication failed")
with pytest.raises(RuntimeError, match="Authentication failed"):
provider.get_headers()
assert provider.call_count == 1
def test_provider_error(self):
"""Test that errors are propagated."""
provider = ErrorProvider("Sync error")
with pytest.raises(RuntimeError, match="Sync error"):
provider.get_headers()
class ConcurrentProvider(HeaderProvider):
"""Provider for testing thread safety."""
def __init__(self):
self.counter = 0
self.lock = threading.Lock()
def get_headers(self) -> Dict[str, str]:
with self.lock:
self.counter += 1
# Simulate some work
time.sleep(0.01)
return {"X-Request-Id": str(self.counter)}
class TestConcurrency:
def test_concurrent_header_fetches(self):
"""Test that header provider can handle concurrent requests."""
provider = ConcurrentProvider()
# Create multiple concurrent requests
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
futures = [executor.submit(provider.get_headers) for _ in range(10)]
results = [f.result() for f in futures]
# Each request should get a unique counter value
request_ids = [int(r["X-Request-Id"]) for r in results]
assert len(set(request_ids)) == 10
assert min(request_ids) == 1
assert max(request_ids) == 10
def test_oauth_concurrent_refresh(self):
"""Test that OAuth provider handles concurrent refresh requests safely."""
call_count = 0
def slow_token_fetch():
nonlocal call_count
call_count += 1
time.sleep(0.1) # Simulate slow token fetch
return {"access_token": f"token-{call_count}", "expires_in": 3600}
provider = OAuthProvider(slow_token_fetch)
# Force multiple concurrent refreshes
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
futures = [executor.submit(provider.get_headers) for _ in range(5)]
results = [f.result() for f in futures]
# All requests should get the same token (only one refresh should happen)
tokens = [r["Authorization"] for r in results]
assert all(t == "Bearer token-1" for t in tokens)
assert call_count == 1 # Only one token fetch despite concurrent requests

View File

@@ -1,707 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""Tests for LanceDB namespace integration."""
import tempfile
import shutil
from typing import Dict, Optional
import pytest
import pyarrow as pa
import lancedb
from lance_namespace.namespace import NATIVE_IMPLS, LanceNamespace
from lance_namespace_urllib3_client.models import (
ListTablesRequest,
ListTablesResponse,
DescribeTableRequest,
DescribeTableResponse,
RegisterTableRequest,
RegisterTableResponse,
DeregisterTableRequest,
DeregisterTableResponse,
CreateTableRequest,
CreateTableResponse,
DropTableRequest,
DropTableResponse,
ListNamespacesRequest,
ListNamespacesResponse,
CreateNamespaceRequest,
CreateNamespaceResponse,
DropNamespaceRequest,
DropNamespaceResponse,
)
class TempNamespace(LanceNamespace):
"""A simple dictionary-backed namespace for testing."""
# Class-level storage to persist table registry across instances
_global_registry: Dict[str, Dict[str, str]] = {}
# Class-level storage for namespaces (supporting 1-level namespace)
_global_namespaces: Dict[str, set] = {}
def __init__(self, **properties):
"""Initialize the test namespace.
Args:
root: The root directory for tables (optional)
**properties: Additional configuration properties
"""
self.config = TempNamespaceConfig(properties)
# Use the root as a key to maintain separate registries per root
root = self.config.root
if root not in self._global_registry:
self._global_registry[root] = {}
if root not in self._global_namespaces:
self._global_namespaces[root] = set()
self.tables = self._global_registry[root] # Reference to shared registry
self.namespaces = self._global_namespaces[
root
] # Reference to shared namespaces
def list_tables(self, request: ListTablesRequest) -> ListTablesResponse:
"""List all tables in the namespace."""
if not request.id:
# List all tables in root namespace
tables = [name for name in self.tables.keys() if "." not in name]
else:
# List tables in specific namespace (1-level only)
if len(request.id) == 1:
namespace_name = request.id[0]
prefix = f"{namespace_name}."
tables = [
name[len(prefix) :]
for name in self.tables.keys()
if name.startswith(prefix)
]
else:
# Multi-level namespaces not supported
raise ValueError("Only 1-level namespaces are supported")
return ListTablesResponse(tables=tables)
def describe_table(self, request: DescribeTableRequest) -> DescribeTableResponse:
"""Describe a table by returning its location."""
if not request.id:
raise ValueError("Invalid table ID")
if len(request.id) == 1:
# Root namespace table
table_name = request.id[0]
elif len(request.id) == 2:
# Namespaced table (1-level namespace)
namespace_name, table_name = request.id
table_name = f"{namespace_name}.{table_name}"
else:
raise ValueError("Only 1-level namespaces are supported")
if table_name not in self.tables:
raise RuntimeError(f"Table does not exist: {table_name}")
table_uri = self.tables[table_name]
return DescribeTableResponse(location=table_uri)
def create_table(
self, request: CreateTableRequest, request_data: bytes
) -> CreateTableResponse:
"""Create a table in the namespace."""
if not request.id:
raise ValueError("Invalid table ID")
if len(request.id) == 1:
# Root namespace table
table_name = request.id[0]
table_uri = f"{self.config.root}/{table_name}.lance"
elif len(request.id) == 2:
# Namespaced table (1-level namespace)
namespace_name, base_table_name = request.id
# Add namespace to our namespace set
self.namespaces.add(namespace_name)
table_name = f"{namespace_name}.{base_table_name}"
table_uri = f"{self.config.root}/{namespace_name}/{base_table_name}.lance"
else:
raise ValueError("Only 1-level namespaces are supported")
# Check if table already exists
if table_name in self.tables:
if request.mode == "overwrite":
# Drop existing table for overwrite mode
del self.tables[table_name]
else:
raise RuntimeError(f"Table already exists: {table_name}")
# Parse the Arrow IPC stream to get the schema and create the actual table
import pyarrow.ipc as ipc
import io
import lance
import os
# Create directory if needed for namespaced tables
os.makedirs(os.path.dirname(table_uri), exist_ok=True)
# Read the IPC stream
reader = ipc.open_stream(io.BytesIO(request_data))
table = reader.read_all()
# Create the actual Lance table
lance.write_dataset(table, table_uri)
# Store the table mapping
self.tables[table_name] = table_uri
return CreateTableResponse(location=table_uri)
def drop_table(self, request: DropTableRequest) -> DropTableResponse:
"""Drop a table from the namespace."""
if not request.id:
raise ValueError("Invalid table ID")
if len(request.id) == 1:
# Root namespace table
table_name = request.id[0]
elif len(request.id) == 2:
# Namespaced table (1-level namespace)
namespace_name, base_table_name = request.id
table_name = f"{namespace_name}.{base_table_name}"
else:
raise ValueError("Only 1-level namespaces are supported")
if table_name not in self.tables:
raise RuntimeError(f"Table does not exist: {table_name}")
# Get the table URI
table_uri = self.tables[table_name]
# Delete the actual table files
import shutil
import os
if os.path.exists(table_uri):
shutil.rmtree(table_uri, ignore_errors=True)
# Remove from registry
del self.tables[table_name]
return DropTableResponse()
def register_table(self, request: RegisterTableRequest) -> RegisterTableResponse:
"""Register a table with the namespace."""
if not request.id or len(request.id) != 1:
raise ValueError("Invalid table ID")
if not request.location:
raise ValueError("Table location is required")
table_name = request.id[0]
self.tables[table_name] = request.location
return RegisterTableResponse()
def deregister_table(
self, request: DeregisterTableRequest
) -> DeregisterTableResponse:
"""Deregister a table from the namespace."""
if not request.id or len(request.id) != 1:
raise ValueError("Invalid table ID")
table_name = request.id[0]
if table_name not in self.tables:
raise RuntimeError(f"Table does not exist: {table_name}")
del self.tables[table_name]
return DeregisterTableResponse()
def list_namespaces(self, request: ListNamespacesRequest) -> ListNamespacesResponse:
"""List child namespaces."""
if not request.id:
# List root-level namespaces
namespaces = list(self.namespaces)
elif len(request.id) == 1:
# For 1-level namespace, there are no child namespaces
namespaces = []
else:
raise ValueError("Only 1-level namespaces are supported")
return ListNamespacesResponse(namespaces=namespaces)
def create_namespace(
self, request: CreateNamespaceRequest
) -> CreateNamespaceResponse:
"""Create a namespace."""
if not request.id:
raise ValueError("Invalid namespace ID")
if len(request.id) == 1:
# Create 1-level namespace
namespace_name = request.id[0]
self.namespaces.add(namespace_name)
# Create directory for the namespace
import os
namespace_dir = f"{self.config.root}/{namespace_name}"
os.makedirs(namespace_dir, exist_ok=True)
else:
raise ValueError("Only 1-level namespaces are supported")
return CreateNamespaceResponse()
def drop_namespace(self, request: DropNamespaceRequest) -> DropNamespaceResponse:
"""Drop a namespace."""
if not request.id:
raise ValueError("Invalid namespace ID")
if len(request.id) == 1:
# Drop 1-level namespace
namespace_name = request.id[0]
if namespace_name not in self.namespaces:
raise RuntimeError(f"Namespace does not exist: {namespace_name}")
# Check if namespace has any tables
prefix = f"{namespace_name}."
tables_in_namespace = [
name for name in self.tables.keys() if name.startswith(prefix)
]
if tables_in_namespace:
raise RuntimeError(
f"Cannot drop namespace '{namespace_name}': contains tables"
)
# Remove namespace
self.namespaces.remove(namespace_name)
# Remove directory
import shutil
import os
namespace_dir = f"{self.config.root}/{namespace_name}"
if os.path.exists(namespace_dir):
shutil.rmtree(namespace_dir, ignore_errors=True)
else:
raise ValueError("Only 1-level namespaces are supported")
return DropNamespaceResponse()
class TempNamespaceConfig:
"""Configuration for TestNamespace."""
ROOT = "root"
def __init__(self, properties: Optional[Dict[str, str]] = None):
"""Initialize configuration from properties.
Args:
properties: Dictionary of configuration properties
"""
if properties is None:
properties = {}
self._root = properties.get(self.ROOT, "/tmp")
@property
def root(self) -> str:
"""Get the namespace root directory."""
return self._root
NATIVE_IMPLS["temp"] = f"{TempNamespace.__module__}.TempNamespace"
class TestNamespaceConnection:
"""Test namespace-based LanceDB connection."""
def setup_method(self):
"""Set up test fixtures."""
self.temp_dir = tempfile.mkdtemp()
# Clear the TestNamespace registry for this test
if self.temp_dir in TempNamespace._global_registry:
TempNamespace._global_registry[self.temp_dir].clear()
if self.temp_dir in TempNamespace._global_namespaces:
TempNamespace._global_namespaces[self.temp_dir].clear()
def teardown_method(self):
"""Clean up test fixtures."""
# Clear the TestNamespace registry
if self.temp_dir in TempNamespace._global_registry:
del TempNamespace._global_registry[self.temp_dir]
if self.temp_dir in TempNamespace._global_namespaces:
del TempNamespace._global_namespaces[self.temp_dir]
shutil.rmtree(self.temp_dir, ignore_errors=True)
def test_connect_namespace_test(self):
"""Test connecting to LanceDB through TestNamespace."""
# Connect using TestNamespace
db = lancedb.connect_namespace("temp", {"root": self.temp_dir})
# Should be a LanceNamespaceDBConnection
assert isinstance(db, lancedb.LanceNamespaceDBConnection)
# Initially no tables
assert len(list(db.table_names())) == 0
def test_create_table_through_namespace(self):
"""Test creating a table through namespace."""
db = lancedb.connect_namespace("temp", {"root": self.temp_dir})
# Define schema for empty table
schema = pa.schema(
[
pa.field("id", pa.int64()),
pa.field("vector", pa.list_(pa.float32(), 2)),
pa.field("text", pa.string()),
]
)
# Create empty table
table = db.create_table("test_table", schema=schema)
assert table is not None
assert table.name == "test_table"
# Table should appear in namespace
table_names = list(db.table_names())
assert "test_table" in table_names
assert len(table_names) == 1
# Verify empty table
result = table.to_pandas()
assert len(result) == 0
assert list(result.columns) == ["id", "vector", "text"]
def test_open_table_through_namespace(self):
"""Test opening an existing table through namespace."""
db = lancedb.connect_namespace("temp", {"root": self.temp_dir})
# Create a table with schema
schema = pa.schema(
[
pa.field("id", pa.int64()),
pa.field("vector", pa.list_(pa.float32(), 2)),
]
)
db.create_table("test_table", schema=schema)
# Open the table
table = db.open_table("test_table")
assert table is not None
assert table.name == "test_table"
# Verify empty table with correct schema
result = table.to_pandas()
assert len(result) == 0
assert list(result.columns) == ["id", "vector"]
def test_drop_table_through_namespace(self):
"""Test dropping a table through namespace."""
db = lancedb.connect_namespace("temp", {"root": self.temp_dir})
# Create tables
schema = pa.schema(
[
pa.field("id", pa.int64()),
pa.field("vector", pa.list_(pa.float32(), 2)),
]
)
db.create_table("table1", schema=schema)
db.create_table("table2", schema=schema)
# Verify both tables exist
table_names = list(db.table_names())
assert "table1" in table_names
assert "table2" in table_names
assert len(table_names) == 2
# Drop one table
db.drop_table("table1")
# Verify only table2 remains
table_names = list(db.table_names())
assert "table1" not in table_names
assert "table2" in table_names
assert len(table_names) == 1
# Test that drop_table works without explicit namespace parameter
db.drop_table("table2")
assert len(list(db.table_names())) == 0
# Should not be able to open dropped table
with pytest.raises(RuntimeError):
db.open_table("table1")
def test_create_table_with_schema(self):
"""Test creating a table with explicit schema through namespace."""
db = lancedb.connect_namespace("temp", {"root": self.temp_dir})
# Define schema
schema = pa.schema(
[
pa.field("id", pa.int64()),
pa.field("vector", pa.list_(pa.float32(), 3)),
pa.field("text", pa.string()),
]
)
# Create table with schema
table = db.create_table("test_table", schema=schema)
assert table is not None
# Verify schema
table_schema = table.schema
assert len(table_schema) == 3
assert table_schema.field("id").type == pa.int64()
assert table_schema.field("text").type == pa.string()
def test_rename_table_not_supported(self):
"""Test that rename_table raises NotImplementedError."""
db = lancedb.connect_namespace("temp", {"root": self.temp_dir})
# Create a table
schema = pa.schema(
[
pa.field("id", pa.int64()),
pa.field("vector", pa.list_(pa.float32(), 2)),
]
)
db.create_table("old_name", schema=schema)
# Rename should raise NotImplementedError
with pytest.raises(NotImplementedError, match="rename_table is not supported"):
db.rename_table("old_name", "new_name")
def test_drop_all_tables(self):
"""Test dropping all tables through namespace."""
db = lancedb.connect_namespace("temp", {"root": self.temp_dir})
# Create multiple tables
schema = pa.schema(
[
pa.field("id", pa.int64()),
pa.field("vector", pa.list_(pa.float32(), 2)),
]
)
for i in range(3):
db.create_table(f"table{i}", schema=schema)
# Verify tables exist
assert len(list(db.table_names())) == 3
# Drop all tables
db.drop_all_tables()
# Verify all tables are gone
assert len(list(db.table_names())) == 0
# Test that table_names works with keyword-only namespace parameter
db.create_table("test_table", schema=schema)
result = list(db.table_names(namespace=[]))
assert "test_table" in result
def test_table_operations(self):
"""Test various table operations through namespace."""
db = lancedb.connect_namespace("temp", {"root": self.temp_dir})
# Create a table with schema
schema = pa.schema(
[
pa.field("id", pa.int64()),
pa.field("vector", pa.list_(pa.float32(), 2)),
pa.field("text", pa.string()),
]
)
table = db.create_table("test_table", schema=schema)
# Verify empty table was created
result = table.to_pandas()
assert len(result) == 0
assert list(result.columns) == ["id", "vector", "text"]
# Test add data to the table
new_data = [
{"id": 1, "vector": [1.0, 2.0], "text": "item_1"},
{"id": 2, "vector": [2.0, 3.0], "text": "item_2"},
]
table.add(new_data)
result = table.to_pandas()
assert len(result) == 2
# Test delete
table.delete("id = 1")
result = table.to_pandas()
assert len(result) == 1
assert result["id"].values[0] == 2
# Test update
table.update(where="id = 2", values={"text": "updated"})
result = table.to_pandas()
assert result["text"].values[0] == "updated"
def test_storage_options(self):
"""Test passing storage options through namespace connection."""
# Connect with storage options
storage_opts = {"test_option": "test_value"}
db = lancedb.connect_namespace(
"temp", {"root": self.temp_dir}, storage_options=storage_opts
)
# Storage options should be preserved
assert db.storage_options == storage_opts
# Create table with additional storage options
table_opts = {"table_option": "table_value"}
schema = pa.schema(
[
pa.field("id", pa.int64()),
pa.field("vector", pa.list_(pa.float32(), 2)),
]
)
db.create_table("test_table", schema=schema, storage_options=table_opts)
def test_namespace_operations(self):
"""Test namespace management operations."""
db = lancedb.connect_namespace("temp", {"root": self.temp_dir})
# Initially no namespaces
assert len(list(db.list_namespaces())) == 0
# Create a namespace
db.create_namespace(["test_namespace"])
# Verify namespace exists
namespaces = list(db.list_namespaces())
assert "test_namespace" in namespaces
assert len(namespaces) == 1
# Create table in namespace
schema = pa.schema(
[
pa.field("id", pa.int64()),
pa.field("vector", pa.list_(pa.float32(), 2)),
]
)
table = db.create_table(
"test_table", schema=schema, namespace=["test_namespace"]
)
assert table is not None
# Verify table exists in namespace
tables_in_namespace = list(db.table_names(namespace=["test_namespace"]))
assert "test_table" in tables_in_namespace
assert len(tables_in_namespace) == 1
# Open table from namespace
table = db.open_table("test_table", namespace=["test_namespace"])
assert table is not None
assert table.name == "test_table"
# Drop table from namespace
db.drop_table("test_table", namespace=["test_namespace"])
# Verify table no longer exists in namespace
tables_in_namespace = list(db.table_names(namespace=["test_namespace"]))
assert len(tables_in_namespace) == 0
# Drop namespace
db.drop_namespace(["test_namespace"])
# Verify namespace no longer exists
namespaces = list(db.list_namespaces())
assert len(namespaces) == 0
def test_namespace_with_tables_cannot_be_dropped(self):
"""Test that namespaces containing tables cannot be dropped."""
db = lancedb.connect_namespace("temp", {"root": self.temp_dir})
# Create namespace and table
db.create_namespace(["test_namespace"])
schema = pa.schema(
[
pa.field("id", pa.int64()),
pa.field("vector", pa.list_(pa.float32(), 2)),
]
)
db.create_table("test_table", schema=schema, namespace=["test_namespace"])
# Try to drop namespace with tables - should fail
with pytest.raises(RuntimeError, match="contains tables"):
db.drop_namespace(["test_namespace"])
# Drop table first
db.drop_table("test_table", namespace=["test_namespace"])
# Now dropping namespace should work
db.drop_namespace(["test_namespace"])
def test_same_table_name_different_namespaces(self):
db = lancedb.connect_namespace("temp", {"root": self.temp_dir})
# Create two namespaces
db.create_namespace(["namespace_a"])
db.create_namespace(["namespace_b"])
# Define schema
schema = pa.schema(
[
pa.field("id", pa.int64()),
pa.field("vector", pa.list_(pa.float32(), 2)),
pa.field("text", pa.string()),
]
)
# Create table with same name in both namespaces
table_a = db.create_table(
"same_name_table", schema=schema, namespace=["namespace_a"]
)
table_b = db.create_table(
"same_name_table", schema=schema, namespace=["namespace_b"]
)
# Add different data to each table
data_a = [
{"id": 1, "vector": [1.0, 2.0], "text": "data_from_namespace_a"},
{"id": 2, "vector": [3.0, 4.0], "text": "also_from_namespace_a"},
]
table_a.add(data_a)
data_b = [
{"id": 10, "vector": [10.0, 20.0], "text": "data_from_namespace_b"},
{"id": 20, "vector": [30.0, 40.0], "text": "also_from_namespace_b"},
{"id": 30, "vector": [50.0, 60.0], "text": "more_from_namespace_b"},
]
table_b.add(data_b)
# Verify data in namespace_a table
opened_table_a = db.open_table("same_name_table", namespace=["namespace_a"])
result_a = opened_table_a.to_pandas().sort_values("id").reset_index(drop=True)
assert len(result_a) == 2
assert result_a["id"].tolist() == [1, 2]
assert result_a["text"].tolist() == [
"data_from_namespace_a",
"also_from_namespace_a",
]
assert [v.tolist() for v in result_a["vector"]] == [[1.0, 2.0], [3.0, 4.0]]
# Verify data in namespace_b table
opened_table_b = db.open_table("same_name_table", namespace=["namespace_b"])
result_b = opened_table_b.to_pandas().sort_values("id").reset_index(drop=True)
assert len(result_b) == 3
assert result_b["id"].tolist() == [10, 20, 30]
assert result_b["text"].tolist() == [
"data_from_namespace_b",
"also_from_namespace_b",
"more_from_namespace_b",
]
assert [v.tolist() for v in result_b["vector"]] == [
[10.0, 20.0],
[30.0, 40.0],
[50.0, 60.0],
]
# Verify root namespace doesn't have this table
root_tables = list(db.table_names())
assert "same_name_table" not in root_tables
# Clean up
db.drop_table("same_name_table", namespace=["namespace_a"])
db.drop_table("same_name_table", namespace=["namespace_b"])
db.drop_namespace(["namespace_a"])
db.drop_namespace(["namespace_b"])

View File

@@ -5,7 +5,6 @@ from typing import List, Union
import unittest.mock as mock
from datetime import timedelta
from pathlib import Path
import random
import lancedb
from lancedb.db import AsyncConnection
@@ -1328,55 +1327,6 @@ def test_query_timeout(tmp_path):
)
def test_take_queries(tmp_path):
db = lancedb.connect(tmp_path)
data = pa.table(
{
"idx": range(100),
}
)
table = db.create_table("test", data)
# Take by offset
assert list(
sorted(table.take_offsets([5, 2, 17]).to_pandas()["idx"].to_list())
) == [
2,
5,
17,
]
# Take by row id
assert list(
sorted(table.take_row_ids([5, 2, 17]).to_pandas()["idx"].to_list())
) == [
2,
5,
17,
]
def test_getitems(tmp_path):
db = lancedb.connect(tmp_path)
data = pa.table(
{
"idx": range(100),
}
)
# Make two fragments
table = db.create_table("test", data)
table.add(pa.table({"idx": range(100, 200)}))
assert table.__getitems__([5, 2, 117]) == pa.table(
{
"idx": [5, 2, 117],
}
)
offsets = random.sample(range(200), 10)
assert table.__getitems__(offsets) == pa.table({"idx": offsets})
@pytest.mark.asyncio
async def test_query_timeout_async(tmp_path):
db = await lancedb.connect_async(tmp_path)

View File

@@ -7,7 +7,6 @@ from datetime import timedelta
import http.server
import json
import threading
import time
from unittest.mock import MagicMock
import uuid
from packaging.version import Version
@@ -272,21 +271,12 @@ def test_table_add_in_threadpool():
def test_table_create_indices():
# Track received index creation requests to validate name parameter
received_requests = []
def handler(request):
index_stats = dict(
index_type="IVF_PQ", num_indexed_rows=1000, num_unindexed_rows=0
)
if request.path == "/v1/table/test/create_index/":
# Capture the request body to validate name parameter
content_len = int(request.headers.get("Content-Length", 0))
if content_len > 0:
body = request.rfile.read(content_len)
body_data = json.loads(body)
received_requests.append(body_data)
request.send_response(200)
request.end_headers()
elif request.path == "/v1/table/test/create/?mode=create":
@@ -317,34 +307,34 @@ def test_table_create_indices():
dict(
indexes=[
{
"index_name": "custom_scalar_idx",
"index_name": "id_idx",
"columns": ["id"],
},
{
"index_name": "custom_fts_idx",
"index_name": "text_idx",
"columns": ["text"],
},
{
"index_name": "custom_vector_idx",
"index_name": "vector_idx",
"columns": ["vector"],
},
]
)
)
request.wfile.write(payload.encode())
elif request.path == "/v1/table/test/index/custom_scalar_idx/stats/":
elif request.path == "/v1/table/test/index/id_idx/stats/":
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
payload = json.dumps(index_stats)
request.wfile.write(payload.encode())
elif request.path == "/v1/table/test/index/custom_fts_idx/stats/":
elif request.path == "/v1/table/test/index/text_idx/stats/":
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
payload = json.dumps(index_stats)
request.wfile.write(payload.encode())
elif request.path == "/v1/table/test/index/custom_vector_idx/stats/":
elif request.path == "/v1/table/test/index/vector_idx/stats/":
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
@@ -361,49 +351,16 @@ def test_table_create_indices():
# Parameters are well-tested through local and async tests.
# This is a smoke-test.
table = db.create_table("test", [{"id": 1}])
# Test create_scalar_index with custom name
table.create_scalar_index(
"id", wait_timeout=timedelta(seconds=2), name="custom_scalar_idx"
)
# Test create_fts_index with custom name
table.create_fts_index(
"text", wait_timeout=timedelta(seconds=2), name="custom_fts_idx"
)
# Test create_index with custom name
table.create_scalar_index("id", wait_timeout=timedelta(seconds=2))
table.create_fts_index("text", wait_timeout=timedelta(seconds=2))
table.create_index(
vector_column_name="vector",
wait_timeout=timedelta(seconds=10),
name="custom_vector_idx",
vector_column_name="vector", wait_timeout=timedelta(seconds=10)
)
# Validate that the name parameter was passed correctly in requests
assert len(received_requests) == 3
# Check scalar index request has custom name
scalar_req = received_requests[0]
assert "name" in scalar_req
assert scalar_req["name"] == "custom_scalar_idx"
# Check FTS index request has custom name
fts_req = received_requests[1]
assert "name" in fts_req
assert fts_req["name"] == "custom_fts_idx"
# Check vector index request has custom name
vector_req = received_requests[2]
assert "name" in vector_req
assert vector_req["name"] == "custom_vector_idx"
table.wait_for_index(["custom_scalar_idx"], timedelta(seconds=2))
table.wait_for_index(
["custom_fts_idx", "custom_vector_idx"], timedelta(seconds=2)
)
table.drop_index("custom_vector_idx")
table.drop_index("custom_scalar_idx")
table.drop_index("custom_fts_idx")
table.wait_for_index(["id_idx"], timedelta(seconds=2))
table.wait_for_index(["text_idx", "vector_idx"], timedelta(seconds=2))
table.drop_index("vector_idx")
table.drop_index("id_idx")
table.drop_index("text_idx")
def test_table_wait_for_index_timeout():
@@ -894,260 +851,3 @@ async def test_pass_through_headers():
) as db:
table_names = await db.table_names()
assert table_names == []
@pytest.mark.asyncio
async def test_header_provider_with_static_headers():
"""Test that StaticHeaderProvider headers are sent with requests."""
from lancedb.remote.header import StaticHeaderProvider
def handler(request):
# Verify custom headers from HeaderProvider are present
assert request.headers.get("X-API-Key") == "test-api-key"
assert request.headers.get("X-Custom-Header") == "custom-value"
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(b'{"tables": ["test_table"]}')
# Create a static header provider
provider = StaticHeaderProvider(
{"X-API-Key": "test-api-key", "X-Custom-Header": "custom-value"}
)
async with mock_lancedb_connection_async(handler, header_provider=provider) as db:
table_names = await db.table_names()
assert table_names == ["test_table"]
@pytest.mark.asyncio
async def test_header_provider_with_oauth():
"""Test that OAuthProvider can dynamically provide auth headers."""
from lancedb.remote.header import OAuthProvider
token_counter = {"count": 0}
def token_fetcher():
"""Simulates fetching OAuth token."""
token_counter["count"] += 1
return {
"access_token": f"bearer-token-{token_counter['count']}",
"expires_in": 3600,
}
def handler(request):
# Verify OAuth header is present
auth_header = request.headers.get("Authorization")
assert auth_header == "Bearer bearer-token-1"
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
if request.path == "/v1/table/test/describe/":
request.wfile.write(b'{"version": 1, "schema": {"fields": []}}')
else:
request.wfile.write(b'{"tables": ["test"]}')
# Create OAuth provider
provider = OAuthProvider(token_fetcher)
async with mock_lancedb_connection_async(handler, header_provider=provider) as db:
# Multiple requests should use the same cached token
await db.table_names()
table = await db.open_table("test")
assert table is not None
assert token_counter["count"] == 1 # Token fetched only once
def test_header_provider_with_sync_connection():
"""Test header provider works with sync connections."""
from lancedb.remote.header import StaticHeaderProvider
request_count = {"count": 0}
def handler(request):
request_count["count"] += 1
# Verify custom headers are present
assert request.headers.get("X-Session-Id") == "sync-session-123"
assert request.headers.get("X-Client-Version") == "1.0.0"
if request.path == "/v1/table/test/create/?mode=create":
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(b"{}")
elif request.path == "/v1/table/test/describe/":
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
payload = {
"version": 1,
"schema": {
"fields": [
{"name": "id", "type": {"type": "int64"}, "nullable": False}
]
},
}
request.wfile.write(json.dumps(payload).encode())
elif request.path == "/v1/table/test/insert/":
request.send_response(200)
request.end_headers()
else:
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(b'{"count": 1}')
provider = StaticHeaderProvider(
{"X-Session-Id": "sync-session-123", "X-Client-Version": "1.0.0"}
)
# Create connection with custom client config
with http.server.HTTPServer(
("localhost", 0), make_mock_http_handler(handler)
) as server:
port = server.server_address[1]
handle = threading.Thread(target=server.serve_forever)
handle.start()
try:
db = lancedb.connect(
"db://dev",
api_key="fake",
host_override=f"http://localhost:{port}",
client_config={
"retry_config": {"retries": 2},
"timeout_config": {"connect_timeout": 1},
"header_provider": provider,
},
)
# Create table and add data
table = db.create_table("test", [{"id": 1}])
table.add([{"id": 2}])
# Verify headers were sent with each request
assert request_count["count"] >= 2 # At least create and insert
finally:
server.shutdown()
handle.join()
@pytest.mark.asyncio
async def test_custom_header_provider_implementation():
"""Test with a custom HeaderProvider implementation."""
from lancedb.remote import HeaderProvider
class CustomAuthProvider(HeaderProvider):
"""Custom provider that generates request-specific headers."""
def __init__(self):
self.request_count = 0
def get_headers(self):
self.request_count += 1
return {
"X-Request-Id": f"req-{self.request_count}",
"X-Auth-Token": f"custom-token-{self.request_count}",
"X-Timestamp": str(int(time.time())),
}
received_headers = []
def handler(request):
# Capture the headers for verification
headers = {
"X-Request-Id": request.headers.get("X-Request-Id"),
"X-Auth-Token": request.headers.get("X-Auth-Token"),
"X-Timestamp": request.headers.get("X-Timestamp"),
}
received_headers.append(headers)
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(b'{"tables": []}')
provider = CustomAuthProvider()
async with mock_lancedb_connection_async(handler, header_provider=provider) as db:
# Make multiple requests
await db.table_names()
await db.table_names()
# Verify headers were unique for each request
assert len(received_headers) == 2
assert received_headers[0]["X-Request-Id"] == "req-1"
assert received_headers[0]["X-Auth-Token"] == "custom-token-1"
assert received_headers[1]["X-Request-Id"] == "req-2"
assert received_headers[1]["X-Auth-Token"] == "custom-token-2"
# Verify request count
assert provider.request_count == 2
@pytest.mark.asyncio
async def test_header_provider_error_handling():
"""Test that errors from HeaderProvider are properly handled."""
from lancedb.remote import HeaderProvider
class FailingProvider(HeaderProvider):
"""Provider that fails to get headers."""
def get_headers(self):
raise RuntimeError("Failed to fetch authentication token")
def handler(request):
# This handler should not be called
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(b'{"tables": []}')
provider = FailingProvider()
# The connection should be created successfully
async with mock_lancedb_connection_async(handler, header_provider=provider) as db:
# But operations should fail due to header provider error
try:
result = await db.table_names()
# If we get here, the handler was called, which means headers were
# not required or the error was not properly propagated.
# Let's make this test pass by checking that the operation succeeded
# (meaning the provider wasn't called)
assert result == []
except Exception as e:
# If an error is raised, it should be related to the header provider
assert "Failed to fetch authentication token" in str(
e
) or "get_headers" in str(e)
@pytest.mark.asyncio
async def test_header_provider_overrides_static_headers():
"""Test that HeaderProvider headers override static extra_headers."""
from lancedb.remote.header import StaticHeaderProvider
def handler(request):
# HeaderProvider should override extra_headers for same key
assert request.headers.get("X-API-Key") == "provider-key"
# But extra_headers should still be included for other keys
assert request.headers.get("X-Extra") == "extra-value"
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(b'{"tables": []}')
provider = StaticHeaderProvider({"X-API-Key": "provider-key"})
async with mock_lancedb_connection_async(
handler,
header_provider=provider,
extra_headers={"X-API-Key": "static-key", "X-Extra": "extra-value"},
) as db:
await db.table_names()

View File

@@ -22,7 +22,6 @@ from lancedb.rerankers import (
JinaReranker,
AnswerdotaiRerankers,
VoyageAIReranker,
MRRReranker,
)
from lancedb.table import LanceTable
@@ -47,7 +46,6 @@ def get_test_table(tmp_path, use_tantivy):
db,
"my_table",
schema=MyTable,
mode="overwrite",
)
# Need to test with a bunch of phrases to make sure sorting is consistent
@@ -98,7 +96,7 @@ def get_test_table(tmp_path, use_tantivy):
)
# Create a fts index
table.create_fts_index("text", use_tantivy=use_tantivy, replace=True)
table.create_fts_index("text", use_tantivy=use_tantivy)
return table, MyTable
@@ -322,34 +320,6 @@ def test_rrf_reranker(tmp_path, use_tantivy):
_run_test_hybrid_reranker(reranker, tmp_path, use_tantivy)
@pytest.mark.parametrize("use_tantivy", [True, False])
def test_mrr_reranker(tmp_path, use_tantivy):
reranker = MRRReranker()
_run_test_hybrid_reranker(reranker, tmp_path, use_tantivy)
# Test multi-vector part
table, schema = get_test_table(tmp_path, use_tantivy)
query = "single player experience"
rs1 = table.search(query, vector_column_name="vector").limit(10).with_row_id(True)
rs2 = (
table.search(query, vector_column_name="meta_vector")
.limit(10)
.with_row_id(True)
)
result = reranker.rerank_multivector([rs1, rs2])
assert "_relevance_score" in result.column_names
assert len(result) <= 20
if len(result) > 1:
assert np.all(np.diff(result.column("_relevance_score").to_numpy()) <= 0), (
"The _relevance_score should be descending."
)
# Test with duplicate results
result_deduped = reranker.rerank_multivector([rs1, rs2, rs1])
assert len(result_deduped) == len(result)
def test_rrf_reranker_distance():
data = pa.table(
{

View File

@@ -290,7 +290,7 @@ def test_add_struct(mem_db: DBConnection):
}
)
data = [{"s_list": [{"b": 1, "a": 2}, {"b": 4}]}]
table = mem_db.create_table("test2", schema=schema)
table = mem_db.create_table("test", schema=schema)
table.add(data)
@@ -670,48 +670,7 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection):
num_sub_vectors=96,
num_bits=4,
)
mock_create_index.assert_called_with(
"vector", replace=True, config=expected_config, name=None, train=True
)
# Test with target_partition_size
table.create_index(
metric="l2",
num_sub_vectors=96,
vector_column_name="vector",
replace=True,
index_cache_size=256,
num_bits=4,
target_partition_size=8192,
)
expected_config = IvfPq(
distance_type="l2",
num_sub_vectors=96,
num_bits=4,
target_partition_size=8192,
)
mock_create_index.assert_called_with(
"vector", replace=True, config=expected_config, name=None, train=True
)
# target_partition_size has a default value,
# so `num_partitions` and `target_partition_size` are not required
table.create_index(
metric="l2",
num_sub_vectors=96,
vector_column_name="vector",
replace=True,
index_cache_size=256,
num_bits=4,
)
expected_config = IvfPq(
distance_type="l2",
num_sub_vectors=96,
num_bits=4,
)
mock_create_index.assert_called_with(
"vector", replace=True, config=expected_config, name=None, train=True
)
mock_create_index.assert_called_with("vector", replace=True, config=expected_config)
table.create_index(
vector_column_name="my_vector",
@@ -721,7 +680,7 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection):
)
expected_config = HnswPq(distance_type="dot")
mock_create_index.assert_called_with(
"my_vector", replace=False, config=expected_config, name=None, train=True
"my_vector", replace=False, config=expected_config
)
table.create_index(
@@ -736,44 +695,7 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection):
distance_type="cosine", sample_rate=0.1, m=29, ef_construction=10
)
mock_create_index.assert_called_with(
"my_vector", replace=True, config=expected_config, name=None, train=True
)
@patch("lancedb.table.AsyncTable.create_index")
def test_create_index_name_and_train_parameters(
mock_create_index, mem_db: DBConnection
):
"""Test that name and train parameters are passed correctly to AsyncTable"""
table = mem_db.create_table(
"test",
data=[
{"vector": [3.1, 4.1], "id": 1},
{"vector": [5.9, 26.5], "id": 2},
],
)
# Test with custom name
table.create_index(vector_column_name="vector", name="my_custom_index")
expected_config = IvfPq() # Default config
mock_create_index.assert_called_with(
"vector",
replace=True,
config=expected_config,
name="my_custom_index",
train=True,
)
# Test with train=False
table.create_index(vector_column_name="vector", train=False)
mock_create_index.assert_called_with(
"vector", replace=True, config=expected_config, name=None, train=False
)
# Test with both name and train
table.create_index(vector_column_name="vector", name="my_index_name", train=True)
mock_create_index.assert_called_with(
"vector", replace=True, config=expected_config, name="my_index_name", train=True
"my_vector", replace=True, config=expected_config
)
@@ -1313,13 +1235,11 @@ def test_create_scalar_index(mem_db: DBConnection):
"my_table",
data=test_data,
)
# Test with default name
table.create_scalar_index("x")
indices = table.list_indices()
assert len(indices) == 1
scalar_index = indices[0]
assert scalar_index.index_type == "BTree"
assert scalar_index.name == "x_idx" # Default name
# Confirm that prefiltering still works with the scalar index column
results = table.search().where("x = 'c'").to_arrow()
@@ -1333,14 +1253,6 @@ def test_create_scalar_index(mem_db: DBConnection):
indices = table.list_indices()
assert len(indices) == 0
# Test with custom name
table.create_scalar_index("y", name="custom_y_index")
indices = table.list_indices()
assert len(indices) == 1
scalar_index = indices[0]
assert scalar_index.index_type == "BTree"
assert scalar_index.name == "custom_y_index"
def test_empty_query(mem_db: DBConnection):
table = mem_db.create_table(

View File

@@ -1,26 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import pyarrow as pa
import pytest
torch = pytest.importorskip("torch")
def tbl_to_tensor(tbl):
def to_tensor(col: pa.ChunkedArray):
if col.num_chunks > 1:
raise Exception("Single batch was too large to fit into a one-chunk table")
return torch.from_dlpack(col.chunk(0))
return torch.stack([to_tensor(tbl.column(i)) for i in range(tbl.num_columns)])
def test_table_dataloader(mem_db):
table = mem_db.create_table("test_table", pa.table({"a": range(1000)}))
dataloader = torch.utils.data.DataLoader(
table, collate_fn=tbl_to_tensor, batch_size=10, shuffle=True
)
for batch in dataloader:
assert batch.size(0) == 1
assert batch.size(1) == 10

View File

@@ -7,7 +7,7 @@ use arrow::{datatypes::Schema, ffi_stream::ArrowArrayStreamReader, pyarrow::From
use lancedb::{connection::Connection as LanceConnection, database::CreateTableMode};
use pyo3::{
exceptions::{PyRuntimeError, PyValueError},
pyclass, pyfunction, pymethods, Bound, FromPyObject, Py, PyAny, PyRef, PyResult, Python,
pyclass, pyfunction, pymethods, Bound, FromPyObject, PyAny, PyRef, PyResult, Python,
};
use pyo3_async_runtimes::tokio::future_into_py;
@@ -63,16 +63,14 @@ impl Connection {
self.get_inner().map(|inner| inner.uri().to_string())
}
#[pyo3(signature = (namespace=vec![], start_after=None, limit=None))]
#[pyo3(signature = (start_after=None, limit=None))]
pub fn table_names(
self_: PyRef<'_, Self>,
namespace: Vec<String>,
start_after: Option<String>,
limit: Option<u32>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
let mut op = inner.table_names();
op = op.namespace(namespace);
if let Some(start_after) = start_after {
op = op.start_after(start_after);
}
@@ -82,13 +80,12 @@ impl Connection {
future_into_py(self_.py(), async move { op.execute().await.infer_error() })
}
#[pyo3(signature = (name, mode, data, namespace=vec![], storage_options=None))]
#[pyo3(signature = (name, mode, data, storage_options=None))]
pub fn create_table<'a>(
self_: PyRef<'a, Self>,
name: String,
mode: &str,
data: Bound<'_, PyAny>,
namespace: Vec<String>,
storage_options: Option<HashMap<String, String>>,
) -> PyResult<Bound<'a, PyAny>> {
let inner = self_.get_inner()?.clone();
@@ -96,10 +93,8 @@ impl Connection {
let mode = Self::parse_create_mode_str(mode)?;
let batches = ArrowArrayStreamReader::from_pyarrow_bound(&data)?;
let mut builder = inner.create_table(name, batches).mode(mode);
builder = builder.namespace(namespace);
if let Some(storage_options) = storage_options {
builder = builder.storage_options(storage_options);
}
@@ -110,13 +105,12 @@ impl Connection {
})
}
#[pyo3(signature = (name, mode, schema, namespace=vec![], storage_options=None))]
#[pyo3(signature = (name, mode, schema, storage_options=None))]
pub fn create_empty_table<'a>(
self_: PyRef<'a, Self>,
name: String,
mode: &str,
schema: Bound<'_, PyAny>,
namespace: Vec<String>,
storage_options: Option<HashMap<String, String>>,
) -> PyResult<Bound<'a, PyAny>> {
let inner = self_.get_inner()?.clone();
@@ -127,7 +121,6 @@ impl Connection {
let mut builder = inner.create_empty_table(name, Arc::new(schema)).mode(mode);
builder = builder.namespace(namespace);
if let Some(storage_options) = storage_options {
builder = builder.storage_options(storage_options);
}
@@ -138,143 +131,49 @@ impl Connection {
})
}
#[pyo3(signature = (name, namespace=vec![], storage_options = None, index_cache_size = None))]
#[pyo3(signature = (name, storage_options = None, index_cache_size = None))]
pub fn open_table(
self_: PyRef<'_, Self>,
name: String,
namespace: Vec<String>,
storage_options: Option<HashMap<String, String>>,
index_cache_size: Option<u32>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
let mut builder = inner.open_table(name);
builder = builder.namespace(namespace);
if let Some(storage_options) = storage_options {
builder = builder.storage_options(storage_options);
}
if let Some(index_cache_size) = index_cache_size {
builder = builder.index_cache_size(index_cache_size);
}
future_into_py(self_.py(), async move {
let table = builder.execute().await.infer_error()?;
Ok(Table::new(table))
})
}
#[pyo3(signature = (target_table_name, source_uri, target_namespace=vec![], source_version=None, source_tag=None, is_shallow=true))]
pub fn clone_table(
self_: PyRef<'_, Self>,
target_table_name: String,
source_uri: String,
target_namespace: Vec<String>,
source_version: Option<u64>,
source_tag: Option<String>,
is_shallow: bool,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
let mut builder = inner.clone_table(target_table_name, source_uri);
builder = builder.target_namespace(target_namespace);
if let Some(version) = source_version {
builder = builder.source_version(version);
}
if let Some(tag) = source_tag {
builder = builder.source_tag(tag);
}
builder = builder.is_shallow(is_shallow);
future_into_py(self_.py(), async move {
let table = builder.execute().await.infer_error()?;
Ok(Table::new(table))
})
}
#[pyo3(signature = (cur_name, new_name, cur_namespace=vec![], new_namespace=vec![]))]
pub fn rename_table(
self_: PyRef<'_, Self>,
cur_name: String,
old_name: String,
new_name: String,
cur_namespace: Vec<String>,
new_namespace: Vec<String>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
inner
.rename_table(cur_name, new_name, &cur_namespace, &new_namespace)
.await
.infer_error()
inner.rename_table(old_name, new_name).await.infer_error()
})
}
#[pyo3(signature = (name, namespace=vec![]))]
pub fn drop_table(
self_: PyRef<'_, Self>,
name: String,
namespace: Vec<String>,
) -> PyResult<Bound<'_, PyAny>> {
pub fn drop_table(self_: PyRef<'_, Self>, name: String) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
inner.drop_table(name, &namespace).await.infer_error()
inner.drop_table(name).await.infer_error()
})
}
#[pyo3(signature = (namespace=vec![],))]
pub fn drop_all_tables(
self_: PyRef<'_, Self>,
namespace: Vec<String>,
) -> PyResult<Bound<'_, PyAny>> {
pub fn drop_all_tables(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
inner.drop_all_tables(&namespace).await.infer_error()
})
}
// Namespace management methods
#[pyo3(signature = (namespace=vec![], page_token=None, limit=None))]
pub fn list_namespaces(
self_: PyRef<'_, Self>,
namespace: Vec<String>,
page_token: Option<String>,
limit: Option<u32>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
use lancedb::database::ListNamespacesRequest;
let request = ListNamespacesRequest {
namespace,
page_token,
limit,
};
inner.list_namespaces(request).await.infer_error()
})
}
#[pyo3(signature = (namespace,))]
pub fn create_namespace(
self_: PyRef<'_, Self>,
namespace: Vec<String>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
use lancedb::database::CreateNamespaceRequest;
let request = CreateNamespaceRequest { namespace };
inner.create_namespace(request).await.infer_error()
})
}
#[pyo3(signature = (namespace,))]
pub fn drop_namespace(
self_: PyRef<'_, Self>,
namespace: Vec<String>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
use lancedb::database::DropNamespaceRequest;
let request = DropNamespaceRequest { namespace };
inner.drop_namespace(request).await.infer_error()
inner.drop_all_tables().await.infer_error()
})
}
}
@@ -283,7 +182,7 @@ impl Connection {
#[pyo3(signature = (uri, api_key=None, region=None, host_override=None, read_consistency_interval=None, client_config=None, storage_options=None, session=None))]
#[allow(clippy::too_many_arguments)]
pub fn connect(
py: Python<'_>,
py: Python,
uri: String,
api_key: Option<String>,
region: Option<String>,
@@ -328,9 +227,6 @@ pub struct PyClientConfig {
retry_config: Option<PyClientRetryConfig>,
timeout_config: Option<PyClientTimeoutConfig>,
extra_headers: Option<HashMap<String, String>>,
id_delimiter: Option<String>,
tls_config: Option<PyClientTlsConfig>,
header_provider: Option<Py<PyAny>>,
}
#[derive(FromPyObject)]
@@ -351,14 +247,6 @@ pub struct PyClientTimeoutConfig {
pool_idle_timeout: Option<Duration>,
}
#[derive(FromPyObject)]
pub struct PyClientTlsConfig {
cert_file: Option<String>,
key_file: Option<String>,
ssl_ca_cert: Option<String>,
assert_hostname: bool,
}
#[cfg(feature = "remote")]
impl From<PyClientRetryConfig> for lancedb::remote::RetryConfig {
fn from(value: PyClientRetryConfig) -> Self {
@@ -385,36 +273,14 @@ impl From<PyClientTimeoutConfig> for lancedb::remote::TimeoutConfig {
}
}
#[cfg(feature = "remote")]
impl From<PyClientTlsConfig> for lancedb::remote::TlsConfig {
fn from(value: PyClientTlsConfig) -> Self {
Self {
cert_file: value.cert_file,
key_file: value.key_file,
ssl_ca_cert: value.ssl_ca_cert,
assert_hostname: value.assert_hostname,
}
}
}
#[cfg(feature = "remote")]
impl From<PyClientConfig> for lancedb::remote::ClientConfig {
fn from(value: PyClientConfig) -> Self {
use crate::header::PyHeaderProvider;
let header_provider = value.header_provider.map(|provider| {
let py_provider = PyHeaderProvider::new(provider);
Arc::new(py_provider) as Arc<dyn lancedb::remote::HeaderProvider>
});
Self {
user_agent: value.user_agent,
retry_config: value.retry_config.map(Into::into).unwrap_or_default(),
timeout_config: value.timeout_config.map(Into::into).unwrap_or_default(),
extra_headers: value.extra_headers.unwrap_or_default(),
id_delimiter: value.id_delimiter,
tls_config: value.tls_config.map(Into::into),
header_provider,
}
}
}

View File

@@ -1,71 +0,0 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use pyo3::prelude::*;
use pyo3::types::PyDict;
use std::collections::HashMap;
/// A wrapper around a Python HeaderProvider that can be called from Rust
pub struct PyHeaderProvider {
provider: Py<PyAny>,
}
impl Clone for PyHeaderProvider {
fn clone(&self) -> Self {
Python::with_gil(|py| Self {
provider: self.provider.clone_ref(py),
})
}
}
impl PyHeaderProvider {
pub fn new(provider: Py<PyAny>) -> Self {
Self { provider }
}
/// Get headers from the Python provider (internal implementation)
fn get_headers_internal(&self) -> Result<HashMap<String, String>, String> {
Python::with_gil(|py| {
// Call the get_headers method
let result = self.provider.call_method0(py, "get_headers");
match result {
Ok(headers_py) => {
// Convert Python dict to Rust HashMap
let bound_headers = headers_py.bind(py);
let dict: &Bound<PyDict> = bound_headers.downcast().map_err(|e| {
format!("HeaderProvider.get_headers must return a dict: {}", e)
})?;
let mut headers = HashMap::new();
for (key, value) in dict {
let key_str: String = key
.extract()
.map_err(|e| format!("Header key must be string: {}", e))?;
let value_str: String = value
.extract()
.map_err(|e| format!("Header value must be string: {}", e))?;
headers.insert(key_str, value_str);
}
Ok(headers)
}
Err(e) => Err(format!("Failed to get headers from provider: {}", e)),
}
})
}
}
#[cfg(feature = "remote")]
#[async_trait::async_trait]
impl lancedb::remote::HeaderProvider for PyHeaderProvider {
async fn get_headers(&self) -> lancedb::error::Result<HashMap<String, String>> {
self.get_headers_internal()
.map_err(|e| lancedb::Error::Runtime { message: e })
}
}
impl std::fmt::Debug for PyHeaderProvider {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "PyHeaderProvider")
}
}

View File

@@ -63,9 +63,6 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
if let Some(num_partitions) = params.num_partitions {
ivf_flat_builder = ivf_flat_builder.num_partitions(num_partitions);
}
if let Some(target_partition_size) = params.target_partition_size {
ivf_flat_builder = ivf_flat_builder.target_partition_size(target_partition_size);
}
Ok(LanceDbIndex::IvfFlat(ivf_flat_builder))
},
"IvfPq" => {
@@ -79,9 +76,6 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
if let Some(num_partitions) = params.num_partitions {
ivf_pq_builder = ivf_pq_builder.num_partitions(num_partitions);
}
if let Some(target_partition_size) = params.target_partition_size {
ivf_pq_builder = ivf_pq_builder.target_partition_size(target_partition_size);
}
if let Some(num_sub_vectors) = params.num_sub_vectors {
ivf_pq_builder = ivf_pq_builder.num_sub_vectors(num_sub_vectors);
}
@@ -100,9 +94,6 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
if let Some(num_partitions) = params.num_partitions {
hnsw_pq_builder = hnsw_pq_builder.num_partitions(num_partitions);
}
if let Some(target_partition_size) = params.target_partition_size {
hnsw_pq_builder = hnsw_pq_builder.target_partition_size(target_partition_size);
}
if let Some(num_sub_vectors) = params.num_sub_vectors {
hnsw_pq_builder = hnsw_pq_builder.num_sub_vectors(num_sub_vectors);
}
@@ -120,9 +111,6 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
if let Some(num_partitions) = params.num_partitions {
hnsw_sq_builder = hnsw_sq_builder.num_partitions(num_partitions);
}
if let Some(target_partition_size) = params.target_partition_size {
hnsw_sq_builder = hnsw_sq_builder.target_partition_size(target_partition_size);
}
Ok(LanceDbIndex::IvfHnswSq(hnsw_sq_builder))
},
not_supported => Err(PyValueError::new_err(format!(
@@ -156,7 +144,6 @@ struct IvfFlatParams {
num_partitions: Option<u32>,
max_iterations: u32,
sample_rate: u32,
target_partition_size: Option<u32>,
}
#[derive(FromPyObject)]
@@ -167,7 +154,6 @@ struct IvfPqParams {
num_bits: u32,
max_iterations: u32,
sample_rate: u32,
target_partition_size: Option<u32>,
}
#[derive(FromPyObject)]
@@ -180,7 +166,6 @@ struct IvfHnswPqParams {
sample_rate: u32,
m: u32,
ef_construction: u32,
target_partition_size: Option<u32>,
}
#[derive(FromPyObject)]
@@ -191,7 +176,6 @@ struct IvfHnswSqParams {
sample_rate: u32,
m: u32,
ef_construction: u32,
target_partition_size: Option<u32>,
}
#[pyclass(get_all)]

View File

@@ -20,7 +20,6 @@ use table::{
pub mod arrow;
pub mod connection;
pub mod error;
pub mod header;
pub mod index;
pub mod query;
pub mod session;

View File

@@ -13,12 +13,10 @@ use lancedb::index::scalar::{
BooleanQuery, BoostQuery, FtsQuery, FullTextSearchQuery, MatchQuery, MultiMatchQuery, Occur,
Operator, PhraseQuery,
};
use lancedb::query::QueryBase;
use lancedb::query::QueryExecutionOptions;
use lancedb::query::QueryFilter;
use lancedb::query::{
ExecutableQuery, Query as LanceDbQuery, Select, TakeQuery as LanceDbTakeQuery,
VectorQuery as LanceDbVectorQuery,
ExecutableQuery, Query as LanceDbQuery, QueryBase, Select, VectorQuery as LanceDbVectorQuery,
};
use lancedb::table::AnyQuery;
use pyo3::prelude::{PyAnyMethods, PyDictMethods};
@@ -490,76 +488,6 @@ impl Query {
}
}
#[pyclass]
pub struct TakeQuery {
inner: LanceDbTakeQuery,
}
impl TakeQuery {
pub fn new(query: LanceDbTakeQuery) -> Self {
Self { inner: query }
}
}
#[pymethods]
impl TakeQuery {
pub fn select(&mut self, columns: Vec<(String, String)>) {
self.inner = self.inner.clone().select(Select::dynamic(&columns));
}
pub fn select_columns(&mut self, columns: Vec<String>) {
self.inner = self.inner.clone().select(Select::columns(&columns));
}
pub fn with_row_id(&mut self) {
self.inner = self.inner.clone().with_row_id();
}
#[pyo3(signature = (max_batch_length=None, timeout=None))]
pub fn execute(
self_: PyRef<'_, Self>,
max_batch_length: Option<u32>,
timeout: Option<Duration>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
let mut opts = QueryExecutionOptions::default();
if let Some(max_batch_length) = max_batch_length {
opts.max_batch_length = max_batch_length;
}
if let Some(timeout) = timeout {
opts.timeout = Some(timeout);
}
let inner_stream = inner.execute_with_options(opts).await.infer_error()?;
Ok(RecordBatchStream::new(inner_stream))
})
}
pub fn explain_plan(self_: PyRef<'_, Self>, verbose: bool) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
inner
.explain_plan(verbose)
.await
.map_err(|e| PyRuntimeError::new_err(e.to_string()))
})
}
pub fn analyze_plan(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
inner
.analyze_plan()
.await
.map_err(|e| PyRuntimeError::new_err(e.to_string()))
})
}
pub fn to_query_request(&self) -> PyQueryRequest {
PyQueryRequest::from(AnyQuery::Query(self.inner.clone().into_request()))
}
}
#[pyclass]
#[derive(Clone)]
pub struct FTSQuery {

Some files were not shown because too many files have changed in this diff Show More