rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-09-04 12:38:38 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
aab23eb39ee4e1a2d5898484f35a863d4c8e7036
lancedb/python/pyproject.toml
T
Jack Ye e639b1b650 feat: add asynchronous remote SQL queries (#4070) 
## Summary

Add SQL execution to remote LanceDB connections. On the standard
synchronous connection, `execute_query` waits for the initial result
stream and returns its Arrow reader. `execute_query_async` is called
without Python `await` and immediately returns a query handle for status
inspection, streaming, or cancellation. Local databases report that SQL
is not supported.

The transport and query lifecycle live in Rust. Python exposes
native-backed synchronous and asynchronous connection methods and query
wrappers; it does not use PyArrow's Flight client.

## User experience

The standard synchronous connection supports both direct reads and
background query execution:

```python
db = lancedb.connect(
    "db://analytics",
    api_key="ldb_...",
    sql_host_override="grpc+tls://sql.example.com:10026",
)

# Direct execution waits only until the initial result stream is available.
# Later batches continue streaming as the query progresses.
reader = db.execute_query(
    "SELECT * FROM events",
    default_namespace_path=["production"],
)
for batch in reader:
    print(batch.num_rows)

# Background execution returns a query handle immediately. Despite the
# `_async` suffix, no Python `await` is needed on a synchronous connection.
query = db.execute_query_async("SELECT * FROM events")
print(query.id)

description = db.describe_query(query.id)
print(description.status)
print(description.progress)
print(description.expires_at)

# Start reading as soon as the service advertises partial results. The reader
# continues polling and yields newly available record batches until the query
# and all result endpoints are complete.
reader = query.reader()
for batch in reader:
    print(batch.num_rows)

# Or cancel a different still-running query. Its status becomes "cancelling"
# while the server is still working, then "cancelled" once confirmed.
cancelled_query = db.execute_query_async("SELECT * FROM large_events")
cancelled_query.cancel()
```

The less commonly used asynchronous connection exposes the same
operations as coroutines:

```python
async_db = await lancedb.connect_async(
    "db://analytics",
    api_key="ldb_...",
    sql_host_override="grpc+tls://sql.example.com:10026",
)
query = await async_db.execute_query_async("SELECT * FROM events")
async for batch in await query.reader():
    print(batch.num_rows)
```

The UUIDv7 query id is scoped to the connection that submitted it. The
connection retains lightweight shared query state used by
`query.describe()` and `db.describe_query(query.id)`; the id does not
encode SQL or a Flight continuation token and is not a cross-connection
resume token. Abandoned state has bounded retention, and terminal state
remains available briefly.

Unqualified table names use the connected database and the `public`
namespace by default. `default_namespace_path` accepts a list such as
`["production", "events"]`. SQL can still use qualified names to
reference other databases and namespaces available to the deployment.

## Design

- Uses Arrow Flight `PollFlightInfo` for submission and long polling,
`DoGet` for results, and `CancelFlightInfo` for cancellation. Each
`PollInfo.info` is treated as the cumulative set of currently available
endpoints, so advertised tickets are consumed once and batches can be
delivered before execution is complete.
- Serializes result completion and cancellation into one lifecycle. A
server-accepted request reports `cancelling` and wakes blocked
status/result work; a later retry can confirm `cancelled`. Result
retrieval is rejected after cancellation is accepted, while cancellation
after a result was already delivered is a no-op.
- Assigns a time-ordered UUIDv7 connection-scoped query id and retains
only shared evolving lifecycle state, keeping SQL, Flight continuation
tokens, and Arrow result data out of public ids and the registry.
- Leaves admission control to the server while honoring server
expiration and a local fallback retention window for abandoned entries.
- Retains terminal ids for five minutes so they remain available for
connection-level description.
- Keeps one lazily initialized SQL client on each remote database
connection and attaches fresh authentication, routing, namespace, and
request metadata to every operation.
- Applies the configured overall timeout to each execution, description,
reader, and cancellation operation. A result reader carries one absolute
deadline from `reader()` through the end of streaming; connect and read
timeouts continue to bound their individual phases.
- Returns a bounded, backpressured, single-consumer Arrow stream rather
than collecting the full result in memory. Dropping the reader stops
downloading but does not implicitly cancel the server query.
- Preserves typed schemas for empty result sets through the stream
schema.
- Accepts Flight result messages up to 1 GiB so a valid row containing a
large blob, string, or vector is not rejected by tonic's 4 MiB default
receive limit.
- Supports the Python client first while keeping the authoritative
implementation in the Rust core.
2026-09-03 14:59:14 -07:00

151 lines
4.2 KiB
TOML
Raw Blame History

[project]
name = "lancedb"
# version in Cargo.toml
dynamic = ["version"]
dependencies = [
"deprecation>=2.1.0",
"numpy>=1.24.0",
"overrides>=0.7; python_version<'3.12'",
"packaging>=23.0",
"pyarrow>=16",
"pydantic>=2.7.4,<3",
"tqdm>=4.27.0",
"lance-namespace>=0.3.2"
]
description = "lancedb"
authors = [{ name = "LanceDB Devs", email = "dev@lancedb.com" }]
license = { file = "LICENSE" }
readme = "README.md"
requires-python = ">=3.10"
keywords = [
"data-format",
"data-science",
"machine-learning",
"arrow",
"data-analytics",
]
classifiers = [
"Development Status :: 3 - Alpha",
"Environment :: Console",
"Intended Audience :: Science/Research",
"License :: OSI Approved :: Apache Software License",
"Operating System :: OS Independent",
"Programming Language :: Python",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3 :: Only",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
"Topic :: Scientific/Engineering",
]
[project.urls]
repository = "https://github.com/lancedb/lancedb"
[project.optional-dependencies]
pylance = [
"pylance>=5.0.0b5",
]
# A library only needs the OpenTelemetry API; the application supplies and
# configures the SDK (the actual exporter/reader). See
# https://opentelemetry.io/docs/languages/python/instrumentation/
otel = ["opentelemetry-api"]
tests = [
"aiohttp>=3.9.0",
"boto3>=1.28.57",
"pandas>=1.4",
"pytest>=7.0",
"pytest-mock>=3.10",
"pytest-asyncio>=0.21",
"duckdb>=0.9.0",
"pytz>=2023.3",
"polars>=0.19, <=1.32.3",
"pyarrow<25",
"pyarrow-stubs>=16.0",
"pylance==9.0.0rc1",
"requests>=2.31.0",
"datafusion>=54,<55",
"opentelemetry-sdk>=1.30.0",
]
dev = [
"ruff>=0.3.0",
"pre-commit>=3.5.0",
"pyright>=1.1.350",
'typing-extensions>=4.0.0; python_version < "3.11"',
]
docs = ["mkdocs", "mkdocs-jupyter", "mkdocs-material", "mkdocstrings-python"]
clip = ["torch", "pillow>=12.1.1", "open-clip-torch"]
siglip = ["torch", "pillow>=12.1.1", "transformers>=4.41.0","sentencepiece"]
embeddings = [
"requests>=2.31.0",
"openai>=1.6.1",
"sentence-transformers>=2.2.0",
"torch>=2.0.0",
"pillow>=12.1.1",
"open-clip-torch>=2.20.0",
"cohere>=4.0",
"colpali-engine>=0.3.10",
"huggingface_hub>=0.19.0",
"InstructorEmbedding>=1.0.1",
"google-genai>=1.0.0",
"boto3>=1.28.57",
"awscli>=1.44.38",
"botocore>=1.31.57",
'ibm-watsonx-ai>=1.1.2; python_version >= "3.10"',
"ollama>=0.3.0",
"sentencepiece>=0.1.99"
]
azure = ["adlfs>=2024.2.0"]
[tool.maturin]
python-source = "python"
module-name = "lancedb._lancedb"
# uv installs the project as an editable package before `uv run`, so keep that
# bootstrap build consistent with `maturin develop`.
editable-profile = "dev"
[build-system]
requires = ["maturin>=1.10"]
build-backend = "maturin"
[tool.ruff.lint]
select = ["F", "E", "W", "G", "PERF"]
[tool.pytest.ini_options]
addopts = "--strict-markers --ignore-glob=lancedb/embeddings/*.py"
markers = [
"slow: marks tests as slow (deselect with '-m \"not slow\"')",
"asyncio",
"s3_test",
]
[tool.pyright]
include = [
"python/lancedb/index.py",
"python/lancedb/rerankers/util.py",
"python/lancedb/rerankers/__init__.py",
"python/lancedb/rerankers/voyageai.py",
"python/lancedb/rerankers/jinaai.py",
"python/lancedb/rerankers/openai.py",
"python/lancedb/rerankers/cross_encoder.py",
"python/lancedb/rerankers/colbert.py",
"python/lancedb/rerankers/answerdotai.py",
"python/lancedb/rerankers/cohere.py",
"python/lancedb/arrow.py",
"python/lancedb/__init__.py",
"python/lancedb/types.py",
"python/lancedb/integrations/__init__.py",
"python/lancedb/exceptions.py",
"python/lancedb/background_loop.py",
"python/lancedb/schema.py",
"python/lancedb/sql.py",
"python/lancedb/remote/__init__.py",
"python/lancedb/remote/errors.py",
"python/lancedb/embeddings/__init__.py",
"python/lancedb/_lancedb.pyi",
"python/type_tests/connect.py",
]
exclude = ["python/tests/"]
pythonVersion = "3.13"
Reference in New Issue View Git Blame Copy Permalink