mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-25 15:38:38 +00:00
Merge origin/main into gatekeeper/fix-2325-1
# Conflicts: # python/python/lancedb/__init__.py
This commit is contained in:
Gatefixer
Generated
+2
-2
@@ -1740,9 +1740,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cmov"
|
||||
version = "0.5.3"
|
||||
version = "0.5.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f88a43d011fc4a6876cb7344703e297c71dda42494fee094d5f7c76bf13f746"
|
||||
checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a"
|
||||
|
||||
[[package]]
|
||||
name = "colorchoice"
|
||||
|
||||
+8
-1
@@ -27,6 +27,7 @@ lance-testing = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.15", "git
|
||||
lance-datafusion = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lancedb = { path = "rust/lancedb", default-features = false }
|
||||
ahash = "0.8"
|
||||
# Note that this one does not include pyarrow
|
||||
arrow = { version = "58.0.0", optional = false }
|
||||
@@ -39,6 +40,7 @@ arrow-schema = "58.0.0"
|
||||
arrow-select = "58.0.0"
|
||||
arrow-cast = "58.0.0"
|
||||
async-trait = "0"
|
||||
bytes = "1"
|
||||
datafusion = { version = "54.0.0", default-features = false }
|
||||
datafusion-catalog = "54.0.0"
|
||||
datafusion-common = { version = "54.0.0", default-features = false }
|
||||
@@ -65,7 +67,12 @@ url = "2"
|
||||
num-traits = "0.2"
|
||||
regex = "1.10"
|
||||
semver = "1.0.25"
|
||||
chrono = "0.4"
|
||||
serde = "1"
|
||||
serde_json = "1"
|
||||
tempfile = "3.5.0"
|
||||
tokio = { version = "1.23", features = ["rt-multi-thread", "sync"] }
|
||||
uuid = { version = "1.7.0", features = ["v4"] }
|
||||
chrono = { version = "0.4", default-features = false, features = ["clock"] }
|
||||
|
||||
[profile.ci]
|
||||
debug = "line-tables-only"
|
||||
|
||||
@@ -177,6 +177,11 @@ multiple-versions = "warn"
|
||||
# Wildcard version requirements (`foo = "*"`) are a footgun — they let any
|
||||
# future release in without review. Ban them outright.
|
||||
wildcards = "deny"
|
||||
# Lint every dependency declared by a workspace member against the shared
|
||||
# `[workspace.dependencies]` table: any crate used by more than one member must
|
||||
# go through `workspace = true`, and entries nothing uses are an error. This
|
||||
# keeps versions from drifting between the core crate and the bindings.
|
||||
workspace-dependencies = { duplicates = "deny", unused = "deny" }
|
||||
# Internal workspace crates reference each other via `path = "..."`, which
|
||||
# cargo-deny sees as a wildcard version. That's fine for private workspace
|
||||
# members (not published to crates.io), so allow it specifically for paths.
|
||||
|
||||
@@ -56,6 +56,42 @@ listing a storage directory.
|
||||
|
||||
::: lancedb.LsmWriteSpec
|
||||
|
||||
## Functions and Jobs
|
||||
|
||||
::: lancedb.functions.FunctionArtifact
|
||||
|
||||
::: lancedb.functions.FunctionParameter
|
||||
|
||||
::: lancedb.functions.FunctionResultField
|
||||
|
||||
::: lancedb.functions.FunctionOutput
|
||||
|
||||
::: lancedb.functions.FunctionSignature
|
||||
|
||||
::: lancedb.functions.PythonEnvironmentSpec
|
||||
|
||||
::: lancedb.functions.FunctionVersion
|
||||
|
||||
::: lancedb.functions.PythonRuntimeSpec
|
||||
|
||||
::: lancedb.functions.FunctionVersionRef
|
||||
|
||||
::: lancedb.functions.ApplicationInput
|
||||
|
||||
::: lancedb.functions.FunctionApplication
|
||||
|
||||
::: lancedb.functions.InputBinding
|
||||
|
||||
::: lancedb.functions.OutputMapping
|
||||
|
||||
::: lancedb.functions.FunctionBinding
|
||||
|
||||
::: lancedb.functions.RefreshColumnResult
|
||||
|
||||
::: lancedb.job.Job
|
||||
|
||||
::: lancedb.job.AsyncJob
|
||||
|
||||
## Expressions
|
||||
|
||||
Type-safe expression builder for filters and projections. Use these instead
|
||||
@@ -155,8 +191,9 @@ The same option is available on `lancedb.tokenize(...)` and the deprecated
|
||||
```python
|
||||
import lancedb
|
||||
|
||||
tokens = list(lancedb.tokenize("acme makes searchable data",
|
||||
custom_stop_words=["acme"]))
|
||||
tokens = list(
|
||||
lancedb.tokenize("acme makes searchable data", custom_stop_words=["acme"])
|
||||
)
|
||||
```
|
||||
|
||||
::: lancedb.tokenize
|
||||
|
||||
+4
-4
@@ -16,12 +16,12 @@ crate-type = ["cdylib"]
|
||||
async-trait.workspace = true
|
||||
arrow-ipc.workspace = true
|
||||
arrow-array.workspace = true
|
||||
arrow-buffer = "58.0.0"
|
||||
arrow-buffer.workspace = true
|
||||
half.workspace = true
|
||||
arrow-schema.workspace = true
|
||||
env_logger.workspace = true
|
||||
futures.workspace = true
|
||||
lancedb = { path = "../rust/lancedb", default-features = false }
|
||||
lancedb.workspace = true
|
||||
lance-namespace.workspace = true
|
||||
napi = { version = "3.8.3", default-features = false, features = [
|
||||
"napi9",
|
||||
@@ -29,8 +29,8 @@ napi = { version = "3.8.3", default-features = false, features = [
|
||||
"chrono_date",
|
||||
"serde-json",
|
||||
] }
|
||||
chrono = { version = "0.4", default-features = false, features = ["clock"] }
|
||||
serde_json = "1"
|
||||
chrono.workspace = true
|
||||
serde_json.workspace = true
|
||||
napi-derive = "3.5.2"
|
||||
# Prevent dynamic linking of lzma, which comes from datafusion
|
||||
lzma-sys = { version = "0.1", features = ["static"] }
|
||||
|
||||
@@ -20,18 +20,16 @@ Do NOT assume local-only table helpers exist on remote tables. If the user asks
|
||||
|
||||
1. Identify the SDK: Python, TypeScript, or both.
|
||||
2. Identify the table mode: local/embedded OSS, remote Enterprise/Cloud, or portable across both. If the user says "LanceDB Enterprise", choose the remote table path. If the task involves jobs in any way (listing, inspecting, creating, or canceling jobs), it is always the remote path and requires a remote server connection — see "Connecting to the LanceDB remote server" below before doing anything else.
|
||||
3. Read the matching language branch before writing or changing code:
|
||||
- Python patterns: `references/python/patterns.md`
|
||||
- Python API quick reference: `references/python/api_reference.md`
|
||||
- Python performance guidance: `references/python/performance.md`
|
||||
- TypeScript patterns: `references/typescript/patterns.md`
|
||||
- TypeScript API quick reference: `references/typescript/api_reference.md`
|
||||
- TypeScript performance guidance: `references/typescript/performance.md`
|
||||
3. Read the matching topic reference before writing or changing code:
|
||||
- Column metadata authoring (both SDKs): `references/column_metadata.md`
|
||||
- Branch operations (both SDKs): `references/branch_ops.md`
|
||||
- Remote server connection resolution (jobs, raw REST): `references/remote_connect.md`
|
||||
- Job operations REST API (list/describe/cancel/query_events): `references/remote_jobs.md`
|
||||
4. Start with `patterns.md` for the selected SDK. Read `api_reference.md` when choosing method names or return collectors. Read `performance.md` when the task involves ingestion, indexing, filtering, query tuning, diagnostics, or large datasets. Read `column_metadata.md` when the task is documenting, tagging, classifying, or grouping table columns (field descriptions, `lancedb:tag:*` tags, logical column families). Read `branch_ops.md` when the task involves branch lifecycle (list/create/delete), writing to a non-main branch, or verifying a change stayed off main. Read `remote_connect.md` when the task involves jobs or direct REST access to an Enterprise deployment, and `remote_jobs.md` for the job REST methods themselves (list, describe, cancel, query_events).
|
||||
|
||||
There is no bundled per-language guide. For exact method names, signatures, and options, look them up in the canonical sources instead of relying on memory:
|
||||
- Python: `docs/src/python/python.md` (the hand-maintained API reference) and the source under `python/python/lancedb/` when working inside the LanceDB repo; otherwise <https://lancedb.github.io/lancedb/python/python/>.
|
||||
- TypeScript: the generated typedoc under `docs/src/js/` and the source under `nodejs/lancedb/` when working inside the LanceDB repo; otherwise <https://lancedb.github.io/lancedb/js/globals/>.
|
||||
4. Apply the SDK invariants in "Per-SDK Invariants" below. Read `column_metadata.md` when the task is documenting, tagging, classifying, or grouping table columns (field descriptions, `lancedb:tag:*` tags, logical column families). Read `branch_ops.md` when the task involves branch lifecycle (list/create/delete), writing to a non-main branch, or verifying a change stayed off main. Read `remote_connect.md` when the task involves jobs or direct REST access to an Enterprise deployment, and `remote_jobs.md` for the job REST methods themselves (list, describe, cancel, query_events).
|
||||
5. For Python schemas, favor Pydantic models and validate records before writing. Use PyArrow schemas when Arrow-native, streaming, or highly dynamic data makes them materially better suited.
|
||||
6. Prefer `search()` or `query()` builders with explicit `select()` and `limit()` for reads.
|
||||
7. Avoid table-level full materialization in remote or portable code. This is the main local-vs-remote read pitfall.
|
||||
@@ -54,6 +52,23 @@ The unsafe pattern is table-level or unbounded collection, plus local-only datas
|
||||
- Python: `table.to_pandas()`, `table.to_arrow()`, `table.to_polars()`; `table.to_lance()` is local/OSS-only dataset access, not materialization
|
||||
- TypeScript: `await table.toArrow()`, `await table.query().toArray()` without `limit()`
|
||||
|
||||
## Per-SDK Invariants
|
||||
|
||||
Python:
|
||||
|
||||
- Result collectors: default to `.to_list()` (plain dicts, no extra dependency) or `.to_arrow()` (PyArrow ships with LanceDB). Use `.to_pandas()` / `.to_polars()` only when the project already declares that dependency — do not assume pandas or polars is installed.
|
||||
- Plain scans differ by client: the sync client has no `.query()` method — use `table.search()` with no argument; the async client uses `await async_table.query()`.
|
||||
|
||||
TypeScript:
|
||||
|
||||
- Collect bounded results with `.toArray()` (objects) or `.toArrow()` (Arrow) after `select()` and `limit()`.
|
||||
- For large reads, stream batches instead of collecting: `for await (const batch of table.query().where(...).select(...).limit(...)) { ... }`.
|
||||
|
||||
Both SDKs:
|
||||
|
||||
- Ingest in bulk or in batches of thousands of rows; never write per-row in a loop — each write creates a version and fragment, slowing ingestion and later queries.
|
||||
- Build a vector index once brute-force search is too slow (rule of thumb: beyond roughly 100K vectors locally), and scalar indexes for filtered columns and merge/upsert keys. Use index defaults unless the task states recall/latency requirements.
|
||||
|
||||
## Enterprise: never drop-then-reuse the same table name
|
||||
|
||||
LanceDB Enterprise/Cloud splits a **control plane** (DDL: create/drop/rename) from a **data plane** (query nodes that serve reads). Query nodes cache the resolved dataset for a table name for up to `table_cache_ttl` — **default 300 seconds (5 minutes)**. After you drop or overwrite a table, the control plane updates immediately but the data plane keeps serving the *old* dataset until that cache entry expires. During the window the two planes disagree.
|
||||
|
||||
@@ -1,138 +0,0 @@
|
||||
# Python API Reference
|
||||
|
||||
Quick method reference for Python LanceDB code. Cross-check source for non-trivial claims.
|
||||
|
||||
## Connect
|
||||
|
||||
If you're connecting to a remote database, use this:
|
||||
```python
|
||||
import lancedb
|
||||
|
||||
db = lancedb.connect("db://my-db", api_key=api_key, host_override=host_override) # remote
|
||||
```
|
||||
(values may be found in LANCEDB_API_KEY and LANCEDB_HOST_OVERRIDE, either in env vars or a .env file)
|
||||
|
||||
If you're connecting to a local table using OSS LanceDB, use this:
|
||||
```python
|
||||
db = lancedb.connect("./camelot-db") # local/OSS
|
||||
```
|
||||
If you're not sure which, or if you can't find the api_key or host_override params, ask the user.
|
||||
|
||||
**Place the local database directory next to the script/entrypoint that opens it** (i.e. resolve the path relative to the script, `Path(__file__).parent / "camelot-db"`), not buried under a shared `data/` folder. The Lance dataset is the database, not a data file — keeping it beside its code makes ownership obvious and paths stable regardless of the working directory the script is launched from.
|
||||
|
||||
**Do not name the directory `lancedb`** (e.g. `./lancedb`, `./data/lancedb`). It collides with the imported `lancedb` package name, which is confusing to read and easy to shadow in scripts. Give it a name derived from the repo or dataset with a clear prefix/suffix — for example `./<dataset>-db`, `./<repo>_lancedb`, or `./vectordb`.
|
||||
|
||||
Async:
|
||||
|
||||
```python
|
||||
db = await lancedb.connect_async("./camelot-db")
|
||||
```
|
||||
|
||||
## Table Reads
|
||||
|
||||
| Task | Preferred API |
|
||||
| --- | --- |
|
||||
| Vector search | `table.search(query_vector).limit(k)` |
|
||||
| Full scan with filters/projection (sync) | `table.search().where(...).select(...).limit(...)` |
|
||||
| Full scan with filters/projection (async) | `table.query().where(...).select(...).limit(...)` |
|
||||
| Filter | `.where("col > 10")` |
|
||||
| Projection | `.select(["id", "text"])` |
|
||||
| Bound result count | `.limit(20)` |
|
||||
| Collect bounded result as Python objects (default, no extra deps) | `.to_list()` on query/search result |
|
||||
| Collect bounded result as Arrow (default, `pyarrow` always available) | `.to_arrow()` on query/search result |
|
||||
| Collect bounded result as pandas (only if project uses pandas) | `.to_pandas()` on query/search result |
|
||||
| Collect bounded result as Polars (only if project uses polars) | `.to_polars()` on query/search result |
|
||||
|
||||
## Sync vs Async Scan API
|
||||
|
||||
The plain-scan entry point differs between the sync and async clients. **Verified against `lancedb` 0.34.0** — re-check if the pinned version changes:
|
||||
|
||||
- **Sync** (`lancedb.connect(...)`): the table has **no `.query()` method**. Use `.search()` with no argument for a plain scan; it returns a query builder that supports `.where()`, `.select()`, `.limit()`, and the `.to_list()` / `.to_arrow()` / `.to_pandas()` / `.to_polars()` collectors.
|
||||
```python
|
||||
rows = table.search().where("status = 'ready'").select(["id", "text"]).limit(20).to_list()
|
||||
```
|
||||
- **Async** (`lancedb.connect_async(...)`): the table has **both** `.query()` and `.search()`. Use `.query()` for a plain scan.
|
||||
```python
|
||||
rows = await async_table.query().where("status = 'ready'").select(["id", "text"]).limit(20).to_list()
|
||||
```
|
||||
|
||||
Do not call `table.query()` on a sync table — it raises `AttributeError`.
|
||||
|
||||
## Local vs Remote Table Methods
|
||||
|
||||
| API | Local table | Remote table | Agent guidance |
|
||||
| --- | --- | --- | --- |
|
||||
| `table.search(...)` | Yes | Yes | Preferred read path (sync + async) |
|
||||
| `table.query()` | Async only | Async only | Sync scan path is `table.search()`; `.query()` is the async scan builder |
|
||||
| `table.to_pandas()` | Yes | No / unsafe for portability | Avoid in portable code |
|
||||
| `table.to_arrow()` | Yes | No / unsafe for portability | Avoid in portable code |
|
||||
| `table.to_polars()` | Yes | No / unsafe for portability | Avoid in portable code |
|
||||
| `table.to_lance()` | Yes | No | Local/OSS escape hatch only |
|
||||
|
||||
## Indexes
|
||||
|
||||
Use `create_index(...)` for vector indexes and modern index configs. Use scalar indexes for filtered or merge keys.
|
||||
|
||||
Common calls:
|
||||
|
||||
```python
|
||||
table.create_index("vector")
|
||||
table.create_scalar_index("status")
|
||||
table.create_fts_index("text")
|
||||
```
|
||||
|
||||
Check source docs before specifying advanced index config names or parameters.
|
||||
|
||||
## Filtering And Recall Knobs
|
||||
|
||||
```python
|
||||
table.search(query_vector).where("status = 'ready'") # pre-filter by default
|
||||
table.search(query_vector).where("status = 'ready'", prefilter=False)
|
||||
table.search(query_vector).limit(10).refine_factor(20)
|
||||
table.search(query_vector).limit(10).nprobes(50)
|
||||
```
|
||||
|
||||
Use post-filtering only when fewer than `limit` results are acceptable.
|
||||
|
||||
## Diagnostics
|
||||
|
||||
```python
|
||||
print(table.search(query_vector).where("year > 2000").limit(10).analyze_plan())
|
||||
print(table.index_stats("vector_idx"))
|
||||
```
|
||||
|
||||
Use these before changing indexes or search tuning.
|
||||
|
||||
## Column (Field) Metadata
|
||||
|
||||
```python
|
||||
schema = table.schema # sync property; async: await table.schema()
|
||||
meta = schema.field("category").metadata # dict[bytes, bytes] — Arrow metadata is bytes-keyed
|
||||
res = table.update_field_metadata( # varargs: one dict per field; works local + remote
|
||||
{"path": "category", "metadata": {"lancedb:description": "...", "lancedb:tag:field_type": "label"}}
|
||||
)
|
||||
res.version # new table version
|
||||
```
|
||||
|
||||
Merges by default; a `None` value deletes that key; `"replace": True` swaps the whole map. Nested fields use dot-paths (`"a.b.c"`). `replace_field_metadata` is deprecated. See `references/column_metadata.md` for key conventions (`lancedb:description`, `lancedb:tag:<name>`, `lancedb:logical-column`) and the authoring workflow.
|
||||
|
||||
## Branches
|
||||
|
||||
```python
|
||||
table.branches.list() # non-main branches; {} = only main
|
||||
exp = table.branches.create("exp") # fork off main -> handle scoped to the branch
|
||||
wip = table.branches.checkout("wip") # existing branch -> scoped handle (version= pins read-only)
|
||||
wip = db.open_table("t", branch="wip") # or open scoped directly
|
||||
table.branches.delete("stale") # removes only the branch pointer
|
||||
table.current_branch() # None = main
|
||||
```
|
||||
|
||||
There is no global switch — scoping is per table handle: any read/write on a branch handle lands on that branch; the original handle keeps targeting main. See `references/branch_ops.md` for the model and isolation checks.
|
||||
|
||||
## Maintenance
|
||||
|
||||
```python
|
||||
table.optimize()
|
||||
```
|
||||
|
||||
Call this after every successful local/OSS ingestion. It handles compaction, cleanup of old versions according to retention, and index optimization. Do not add this for LanceDB Enterprise/Cloud remote tables; Enterprise handles compaction and cleanup automatically from cluster configuration.
|
||||
@@ -1,173 +0,0 @@
|
||||
# Python Patterns
|
||||
|
||||
Use these patterns when writing Python code with `lancedb`.
|
||||
|
||||
## Before Writing Code
|
||||
|
||||
Choose the output type from what the project actually depends on. **Do not assume `pandas` or `polars` is installed** — they are heavy dependencies that many LanceDB projects do not use. `pyarrow`, by contrast, ships as a LanceDB dependency and is always available, so it is a safe default to lean on.
|
||||
|
||||
Default output (after applying `select()` and `limit()`):
|
||||
|
||||
- **Python objects**: `.to_list()` — a list of dicts, no extra dependencies. Prefer this for scripts, examples, and agent-generated code unless there is a reason to do otherwise.
|
||||
- **PyArrow**: `.to_arrow()` — a `pyarrow.Table`, when the surrounding code is Arrow-native or you need columnar/zero-copy handoff.
|
||||
|
||||
Only reach for a DataFrame when the project *already* declares that dependency:
|
||||
|
||||
- Pandas projects (pandas in `pyproject.toml`/requirements): `.to_pandas()`.
|
||||
- Polars projects (polars declared): `.to_polars()`.
|
||||
|
||||
If unsure, check the dependency manifest or the imports in surrounding files. When in doubt, use `.to_list()` or `.to_arrow()`.
|
||||
|
||||
## Schema Design and Validation
|
||||
|
||||
Favor `LanceModel` and Pydantic validation for Python schemas. They keep field
|
||||
types readable, validate source records before a write, and map directly to a
|
||||
LanceDB schema. Use `Vector(dimension)` for fixed-size vectors:
|
||||
|
||||
```python
|
||||
from lancedb.pydantic import LanceModel, Vector
|
||||
|
||||
class Document(LanceModel):
|
||||
id: int
|
||||
text: str
|
||||
vector: Vector(384, nullable=False)
|
||||
|
||||
rows = [Document.model_validate(row) for row in source_rows]
|
||||
table = db.create_table("documents", schema=Document)
|
||||
table.add(rows)
|
||||
```
|
||||
|
||||
Use PyArrow schemas instead when the pipeline is already Arrow-native, needs
|
||||
record-batch streaming, or has runtime schema requirements that would make a
|
||||
Pydantic model harder to understand. Declare Pydantic as a direct project
|
||||
dependency when application code imports it, even if LanceDB also depends on it.
|
||||
|
||||
## Recommended Patterns
|
||||
|
||||
### Bounded search or query
|
||||
|
||||
Use this for application reads, examples, notebooks, and agent-generated scripts:
|
||||
|
||||
```python
|
||||
results = (
|
||||
table.search(query_vector)
|
||||
.where("status = 'ready'")
|
||||
.select(["id", "text"])
|
||||
.limit(20)
|
||||
.to_list() # or .to_arrow(); .to_pandas()/.to_polars() only if the project uses them
|
||||
)
|
||||
```
|
||||
|
||||
Why: `search()` works across local and remote tables and on both the sync and async clients. `select()` avoids fetching unused columns. `limit()` prevents accidental full-table reads. `.to_list()` and `.to_arrow()` avoid assuming pandas/polars is installed (see "Before Writing Code").
|
||||
|
||||
For a **plain scan** (no query vector), the entry point differs by client:
|
||||
|
||||
```python
|
||||
# Sync client: no .query() method — use .search() with no argument.
|
||||
rows = table.search().where("status = 'ready'").select(["id", "text"]).limit(20).to_list()
|
||||
|
||||
# Async client: use .query().
|
||||
rows = await async_table.query().where("status = 'ready'").select(["id", "text"]).limit(20).to_list()
|
||||
```
|
||||
|
||||
`table.query()` on a sync table raises `AttributeError` (verified on `lancedb` 0.34.0). See the "Sync vs Async Scan API" section in `api_reference.md`.
|
||||
|
||||
### Bounded query result conversion
|
||||
|
||||
It is fine to collect bounded query/search results:
|
||||
|
||||
```python
|
||||
arrow_table = table.search().select(["id"]).limit(100).to_arrow() # sync plain scan
|
||||
rows = table.search(query_vector).limit(10).to_list()
|
||||
df = table.search(query_vector).limit(10).to_pandas() # only if pandas is a project dep
|
||||
```
|
||||
|
||||
### Local-only Lance dataset API
|
||||
|
||||
`table.to_lance()` does not itself materialize the full dataset. It returns the underlying `lance.LanceDataset`, making the table accessible through the PyLance dataset API. Use it when the task is explicitly local/OSS and needs Lance dataset methods not exposed by LanceDB:
|
||||
|
||||
```python
|
||||
# Local/OSS only: RemoteTable does not expose table.to_lance().
|
||||
ds = table.to_lance()
|
||||
for batch in ds.to_batches(columns=["id", "text"], batch_size=10_000):
|
||||
process(batch)
|
||||
```
|
||||
|
||||
### Async Python
|
||||
|
||||
Keep the same shape and bound the result before collecting:
|
||||
|
||||
```python
|
||||
results = await (
|
||||
async_table.query()
|
||||
.where("status = 'ready'")
|
||||
.select(["id", "text"])
|
||||
.limit(20)
|
||||
.to_list() # or .to_arrow()
|
||||
)
|
||||
```
|
||||
|
||||
## Anti-Patterns
|
||||
|
||||
**Avoid the following anti-patterns in your code.**
|
||||
|
||||
### Table-level full materialization
|
||||
|
||||
Avoid whole-table collectors in portable or large-table code:
|
||||
|
||||
```python
|
||||
df = table.to_pandas()
|
||||
arrow_table = table.to_arrow()
|
||||
polars_df = table.to_polars()
|
||||
```
|
||||
|
||||
Why: local tables expose these whole-table collectors, but remote tables intentionally do not — a remote production table can be far larger than a local development table, so it is easy to accidentally pull the entire table into memory.
|
||||
|
||||
`table.to_lance()` is different: it is not a full materialization call, but it is still local/OSS-only and should not appear in code meant to run against remote Enterprise tables.
|
||||
|
||||
### Unbounded result collection
|
||||
|
||||
Avoid query/search collection without a meaningful limit:
|
||||
|
||||
```python
|
||||
rows = table.search().to_list() # unbounded plain scan
|
||||
rows = table.search(query_vector).to_list() # unbounded vector search
|
||||
```
|
||||
|
||||
Prefer `select(...).limit(...)` before collecting; for large reads, stream in batches instead.
|
||||
|
||||
### Per-row writes
|
||||
|
||||
Avoid loops that write one row per call:
|
||||
|
||||
```python
|
||||
for row in rows:
|
||||
table.add([row]) # one commit + fragment per row
|
||||
```
|
||||
|
||||
Each `add()` creates a new version and fragment. Pass the whole batch in a single call, or chunk very large inputs:
|
||||
|
||||
```python
|
||||
table.add(rows) # single commit
|
||||
# for very large inputs, add batches of several thousand rows
|
||||
```
|
||||
|
||||
After the final successful write to an embedded OSS table, call
|
||||
`table.optimize()`. Skip this for Enterprise/Cloud tables because their
|
||||
maintenance is automatic.
|
||||
|
||||
### Drop-then-reuse the same table name (Enterprise/Cloud)
|
||||
|
||||
Avoid dropping or overwriting a remote table and then reusing that name right away:
|
||||
|
||||
```python
|
||||
db.drop_table("my_table")
|
||||
table = db.create_table("my_table", data=rows) # reads 500 for ~5 min
|
||||
table = db.create_table("my_table", data=rows, mode="overwrite") # same problem
|
||||
```
|
||||
|
||||
Why: Enterprise/Cloud splits DDL (control plane) from query serving (data plane). The data plane caches the dataset behind a table name for up to `table_cache_ttl` (default 300s / 5 min), so after a drop/overwrite the DDL succeeds but queries against the reused name return `500 Internal Server Error` until the cache expires — and a fresh `describe` may still show the old schema. Instead, write to a **fresh name**, use `list_tables()` and fail if it already exists, then `rename_table(fresh, final)` onto the final name only after the old table's drop has propagated (~5 min). See the "Enterprise: never drop-then-reuse the same table name" section in `SKILL.md`. Local/OSS tables have no separate data plane — overwrite freely there.
|
||||
|
||||
### Guessing performance fixes
|
||||
|
||||
Avoid changing `nprobes`, `refine_factor`, or index types before checking the query plan and index stats. Diagnose first, then tune one knob at a time.
|
||||
@@ -1,131 +0,0 @@
|
||||
# Python Performance Guidance
|
||||
|
||||
Use this when writing Python code that ingests data, queries large tables, builds indexes, or investigates latency.
|
||||
|
||||
## Ingestion
|
||||
|
||||
### Recommended: validate schemas and records with Pydantic
|
||||
|
||||
Favor `LanceModel` for readable Python schema definitions and validate source
|
||||
records before writing. Use PyArrow directly for Arrow-native or streaming
|
||||
pipelines where it is the clearer representation.
|
||||
|
||||
```python
|
||||
from lancedb.pydantic import LanceModel, Vector
|
||||
|
||||
class Document(LanceModel):
|
||||
id: int
|
||||
text: str
|
||||
vector: Vector(384, nullable=False)
|
||||
|
||||
rows = [Document.model_validate(row) for row in source_rows]
|
||||
table = db.create_table("documents", schema=Document)
|
||||
table.add(rows)
|
||||
```
|
||||
|
||||
### Recommended: bulk ingestion for materialized data
|
||||
|
||||
```python
|
||||
table.add(arrow_table)
|
||||
table.add(df)
|
||||
table.add(pa.dataset("data/", format="parquet"))
|
||||
```
|
||||
|
||||
For very large initial loads, create the table empty first, then call `add(...)`. Passing data directly to `create_table(name, data)` can skip the auto-parallel write path.
|
||||
|
||||
### Recommended: iterator ingestion for generated or streamed data
|
||||
|
||||
```python
|
||||
def batches():
|
||||
for raw in source:
|
||||
vectors = model.encode(raw["text"])
|
||||
yield pa.RecordBatch.from_pydict({**raw, "vector": vectors})
|
||||
|
||||
table.add(batches())
|
||||
```
|
||||
|
||||
Use chunks of several thousand rows or more when practical. Tiny batches and per-row writes create many small fragments.
|
||||
|
||||
### Anti-pattern: per-row `add()`
|
||||
|
||||
```python
|
||||
for row in rows:
|
||||
table.add([row])
|
||||
```
|
||||
|
||||
Each call creates a version and fragment. This slows ingestion and later queries.
|
||||
|
||||
## Indexing
|
||||
|
||||
- Build a vector index once brute-force vector search becomes too slow. As a rule of thumb, local brute force is fine below roughly 100K vectors; beyond that, build an index.
|
||||
- Use `IVF_PQ` as the general-purpose default. Enterprise builds this automatically.
|
||||
- Use scalar indexes for filtered columns and merge/upsert keys.
|
||||
- Use `BTREE` for mostly distinct numeric/string/temporal columns, `BITMAP` for booleans and low-cardinality columns, and `LABEL_LIST` for list membership queries.
|
||||
- Keep full-text defaults unless phrase queries require position data.
|
||||
|
||||
## Querying
|
||||
|
||||
Always be explicit:
|
||||
|
||||
```python
|
||||
table.search(query_vector).select(["id", "title"]).limit(20)
|
||||
```
|
||||
|
||||
- `select()` reduces bytes read and transferred.
|
||||
- `limit()` prevents accidental full-table materialization.
|
||||
- Pre-filtering is the default and guarantees returned rows satisfy the predicate.
|
||||
- Use post-filtering only when fewer than `limit` results are acceptable.
|
||||
|
||||
## Recall Tuning
|
||||
|
||||
Tune one knob at a time:
|
||||
|
||||
- Quantized indexes: raise `refine_factor` to rescore more candidates on full vectors.
|
||||
- HNSW-backed indexes: raise `ef`; start around `1.5 * k`, increase toward `10 * k` if recall is short.
|
||||
- IVF candidate breadth: `nprobes` is auto-tuned; override only when a selective pre-filter leaves too few neighbors.
|
||||
|
||||
## Maintenance
|
||||
|
||||
After every successful embedded OSS/local ingestion, call `table.optimize()`.
|
||||
Do not add this to LanceDB Enterprise/Cloud remote table code; remote compaction
|
||||
and cleanup are handled automatically based on the Enterprise cluster
|
||||
configuration.
|
||||
|
||||
Why local maintenance is needed:
|
||||
|
||||
- Frequent writes can create many small fragments. Queries then need to scan across more files, which can increase latency.
|
||||
- Updates, deletes, and appends create new table versions. Old versions are retained for time travel and rollback, which can grow disk usage.
|
||||
- Indexes may have newly added rows that are not yet fully optimized into the index structure.
|
||||
|
||||
For local/OSS tables, run `optimize()` after the final successful ingestion
|
||||
write. Also run it after later batches of update/delete operations or on a
|
||||
regular maintenance schedule:
|
||||
|
||||
```python
|
||||
table.optimize()
|
||||
```
|
||||
|
||||
If the user wants more aggressive local disk cleanup, pass a shorter cleanup retention window:
|
||||
|
||||
```python
|
||||
from datetime import timedelta
|
||||
|
||||
table.optimize(cleanup_older_than=timedelta(days=1))
|
||||
```
|
||||
|
||||
Do not use very short cleanup windows when the application depends on time travel, rollback, or old versions.
|
||||
|
||||
## Diagnostics
|
||||
|
||||
Before changing code or indexes, inspect:
|
||||
|
||||
```python
|
||||
print(table.search(query_vector).where("year > 2000").limit(10).analyze_plan())
|
||||
print(table.index_stats("vector_idx"))
|
||||
```
|
||||
|
||||
Look for high scan bytes, missing indexes, fragmented data, and unindexed rows.
|
||||
|
||||
## Python Multiprocessing
|
||||
|
||||
When using multiprocessing, use `spawn` rather than `fork`. LanceDB is multi-threaded internally, and `fork` plus a multi-threaded process is unsafe.
|
||||
@@ -1,105 +0,0 @@
|
||||
# TypeScript API Reference
|
||||
|
||||
Quick method reference for TypeScript LanceDB code. Cross-check source for non-trivial claims.
|
||||
|
||||
## Connect
|
||||
|
||||
```typescript
|
||||
import * as lancedb from "@lancedb/lancedb";
|
||||
|
||||
const db = await lancedb.connect("./camelot-db");
|
||||
```
|
||||
|
||||
**Place the local database directory next to the script/entrypoint that opens it** (resolve the path relative to the module, e.g. via `import.meta.dirname` / `__dirname`), not buried under a shared `data/` folder. The Lance dataset is the database, not a data file — keeping it beside its code makes ownership obvious and paths stable regardless of the working directory the script is launched from.
|
||||
|
||||
**Do not name the directory `lancedb`** (e.g. `./lancedb`, `./data/lancedb`). It collides with the imported `lancedb` package/namespace, which is confusing to read. Give it a name derived from the repo or dataset with a clear prefix/suffix — for example `./<dataset>-db`, `./<repo>_lancedb`, or `./vectordb`.
|
||||
|
||||
Remote connections use `db://...` plus Enterprise/Cloud credentials and deployment settings. Check current source/docs for exact connection options.
|
||||
|
||||
## Table Reads
|
||||
|
||||
| Task | Preferred API |
|
||||
| --- | --- |
|
||||
| Vector search | `table.search(queryVector).limit(k)` |
|
||||
| Full scan with filters/projection | `table.query().where(...).select(...).limit(...)` |
|
||||
| Filter | `.where("col > 10")` |
|
||||
| Projection | `.select(["id", "text"])` |
|
||||
| Bound result count | `.limit(20)` |
|
||||
| Collect bounded result as objects | `.toArray()` on query/search result |
|
||||
| Collect bounded result as Arrow | `.toArrow()` on query/search result |
|
||||
| Stream result batches | `for await (const batch of table.query()...)` |
|
||||
|
||||
## Local vs Remote Safety
|
||||
|
||||
| API | Agent guidance |
|
||||
| --- | --- |
|
||||
| `table.search(...)` | Preferred read path |
|
||||
| `table.query()` | Preferred scan/filter path |
|
||||
| `await table.toArrow()` | Avoid in portable or large-table code |
|
||||
| `await table.query().toArray()` with no `limit()` | Avoid; unbounded collection |
|
||||
| `await table.query().toArrow()` with no `limit()` | Avoid; unbounded collection |
|
||||
|
||||
## Indexes
|
||||
|
||||
```typescript
|
||||
await table.createIndex("vector");
|
||||
await table.createIndex("status");
|
||||
```
|
||||
|
||||
Use vector indexes for large vector search workloads and scalar indexes for filtered columns or merge/upsert keys. Check source/docs before specifying advanced index options.
|
||||
|
||||
## Filtering And Recall Knobs
|
||||
|
||||
```typescript
|
||||
await table.search(queryVector).where("status = 'ready'").limit(10).toArray();
|
||||
await table.search(queryVector).limit(10).refineFactor(20).toArray();
|
||||
await table.search(queryVector).limit(10).nprobes(50).toArray();
|
||||
await table.search(queryVector).limit(10).ef(100).toArray();
|
||||
await table.search(queryVector).where("status = 'ready'").postfilter().limit(10).toArray();
|
||||
```
|
||||
|
||||
Use `postfilter()` only when fewer than `limit` results are acceptable.
|
||||
|
||||
## Diagnostics
|
||||
|
||||
```typescript
|
||||
console.log(await table.search(queryVector).where("year > 2000").limit(10).analyzePlan());
|
||||
console.log(await table.indexStats("vector_idx"));
|
||||
```
|
||||
|
||||
Use these before changing indexes or search tuning.
|
||||
|
||||
## Column (Field) Metadata
|
||||
|
||||
```typescript
|
||||
const schema = await table.schema();
|
||||
const meta = schema.fields.find((f) => f.name === "category")?.metadata; // Map<string, string>
|
||||
const res = await table.updateFieldMetadata([
|
||||
{ path: "category", metadata: { "lancedb:description": "...", "lancedb:tag:field_type": "label" } },
|
||||
]);
|
||||
res.version; // new table version
|
||||
```
|
||||
|
||||
Merges by default; a `null` value deletes that key; `replace: true` swaps the whole map. Nested fields use dot-paths (`"a.b.c"`). See `references/column_metadata.md` for key conventions (`lancedb:description`, `lancedb:tag:<name>`, `lancedb:logical-column`) and the authoring workflow.
|
||||
|
||||
## Branches
|
||||
|
||||
```typescript
|
||||
const branches = await table.branches(); // async manager
|
||||
await branches.list(); // non-main branches; {} = only main
|
||||
const exp = await branches.create("exp"); // fork off main -> Table scoped to the branch
|
||||
const wip = await branches.checkout("wip"); // existing branch -> scoped Table (version arg pins read-only)
|
||||
const wip2 = await db.openTable("t", { branch: "wip" }); // or open scoped directly
|
||||
await branches.delete("stale"); // removes only the branch pointer
|
||||
table.currentBranch(); // null = main
|
||||
```
|
||||
|
||||
There is no global switch — scoping is per table handle: any read/write on a branch handle lands on that branch; the original handle keeps targeting main. See `references/branch_ops.md` for the model and isolation checks.
|
||||
|
||||
## Maintenance
|
||||
|
||||
```typescript
|
||||
await table.optimize();
|
||||
```
|
||||
|
||||
Call this after every successful local/OSS ingestion. It handles compaction, cleanup of old versions according to retention, and index optimization. Do not add this for LanceDB Enterprise/Cloud remote tables; Enterprise handles compaction and cleanup automatically from cluster configuration.
|
||||
@@ -1,100 +0,0 @@
|
||||
# TypeScript Patterns
|
||||
|
||||
Use these patterns when writing TypeScript code with `@lancedb/lancedb`.
|
||||
|
||||
## Recommended Patterns
|
||||
|
||||
### Bounded query
|
||||
|
||||
Use this for application reads, scripts, and examples:
|
||||
|
||||
```typescript
|
||||
const rows = await table
|
||||
.query()
|
||||
.where("status = 'ready'")
|
||||
.select(["id", "text"])
|
||||
.limit(20)
|
||||
.toArray();
|
||||
```
|
||||
|
||||
### Bounded vector search
|
||||
|
||||
```typescript
|
||||
const rows = await table
|
||||
.search(queryVector)
|
||||
.select(["id", "text"])
|
||||
.limit(20)
|
||||
.toArray();
|
||||
```
|
||||
|
||||
### Batch streaming for larger reads
|
||||
|
||||
When the task needs many rows, avoid collecting everything at once:
|
||||
|
||||
```typescript
|
||||
for await (const batch of table
|
||||
.query()
|
||||
.where("status = 'ready'")
|
||||
.select(["id", "text"])
|
||||
.limit(10_000)) {
|
||||
process(batch);
|
||||
}
|
||||
```
|
||||
|
||||
## Anti-Patterns
|
||||
|
||||
**Avoid the following anti-patterns in your code.**
|
||||
|
||||
### Table-level full materialization
|
||||
|
||||
Avoid whole-table collectors in portable or large-table code:
|
||||
|
||||
```typescript
|
||||
const tableArrow = await table.toArrow();
|
||||
```
|
||||
|
||||
Why: local tables expose these whole-table collectors, but remote tables intentionally do not — a remote production table can be far larger than a local development table, so it is easy to accidentally pull the entire table into memory.
|
||||
|
||||
### Unbounded result collection
|
||||
|
||||
Avoid query/search collection without a meaningful limit:
|
||||
|
||||
```typescript
|
||||
const rows = await table.query().toArray(); // unbounded plain scan
|
||||
const rows = await table.search(queryVector).toArray(); // unbounded vector search
|
||||
```
|
||||
|
||||
Prefer `select(...).limit(...)` before collecting; for large reads, stream in batches instead.
|
||||
|
||||
### Per-row writes
|
||||
|
||||
Avoid loops that write one row per call:
|
||||
|
||||
```typescript
|
||||
for (const row of rows) {
|
||||
await table.add([row]); // one commit + fragment per row
|
||||
}
|
||||
```
|
||||
|
||||
Each `add()` creates a new version and fragment. Pass the whole batch in a single call, or chunk very large inputs:
|
||||
|
||||
```typescript
|
||||
await table.add(rows); // single commit
|
||||
// for very large inputs, add in chunks of several thousand rows
|
||||
```
|
||||
|
||||
### Drop-then-reuse the same table name (Enterprise/Cloud)
|
||||
|
||||
Avoid dropping or overwriting a remote table and then reusing that name right away:
|
||||
|
||||
```typescript
|
||||
await db.dropTable("my_table");
|
||||
const table = await db.createTable("my_table", rows); // reads 500 for ~5 min
|
||||
const table = await db.createTable("my_table", rows, { mode: "overwrite" }); // same problem
|
||||
```
|
||||
|
||||
Why: Enterprise/Cloud splits DDL (control plane) from query serving (data plane). The data plane caches the dataset behind a table name for up to `table_cache_ttl` (default 300s / 5 min), so after a drop/overwrite the DDL succeeds but queries against the reused name return `500 Internal Server Error` until the cache expires — and a fresh `describe` may still show the old schema. Instead, write to a **fresh name**, use `tableNames()` and fail if it already exists, then `renameTable(fresh, final)` onto the final name only after the old table's drop has propagated (~5 min). See the "Enterprise: never drop-then-reuse the same table name" section in `SKILL.md`. Local/OSS tables have no separate data plane — overwrite freely there.
|
||||
|
||||
### Guessing performance fixes
|
||||
|
||||
Avoid changing `nprobes`, `refineFactor`, `ef`, or index settings before checking `analyzePlan()` and `indexStats(...)`. Diagnose first, then tune one knob at a time.
|
||||
@@ -1,78 +0,0 @@
|
||||
# TypeScript Performance Guidance
|
||||
|
||||
Use this when writing TypeScript code that ingests data, queries large tables, builds indexes, or investigates latency.
|
||||
|
||||
## Ingestion
|
||||
|
||||
- Prefer bulk or batched writes.
|
||||
- Avoid per-row write loops; they create many small commits/fragments.
|
||||
- For generated data, accumulate reasonable batches before adding.
|
||||
- For file-backed data, prefer APIs that stream from Arrow/Parquet-style inputs when available.
|
||||
|
||||
## Indexing
|
||||
|
||||
- Build a vector index once brute-force vector search becomes too slow. As a rule of thumb, local brute force is fine below roughly 100K vectors; beyond that, build an index.
|
||||
- Use the general-purpose vector index defaults unless the task has explicit recall/latency requirements.
|
||||
- Build scalar indexes for filtered columns and merge/upsert keys.
|
||||
- Use full-text index phrase options only when phrase queries require them.
|
||||
|
||||
## Querying
|
||||
|
||||
Always be explicit:
|
||||
|
||||
```typescript
|
||||
await table.search(queryVector).select(["id", "title"]).limit(20).toArray();
|
||||
```
|
||||
|
||||
- `select()` reduces bytes read and transferred.
|
||||
- `limit()` prevents accidental full-table collection.
|
||||
- Pre-filtering is the default behavior. Use `postfilter()` only when fewer than `limit` results are acceptable.
|
||||
|
||||
## Recall Tuning
|
||||
|
||||
Tune one knob at a time:
|
||||
|
||||
- Quantized indexes: raise `refineFactor(...)` to rescore more candidates on full vectors.
|
||||
- HNSW-backed indexes: raise `ef(...)`; start around `1.5 * k`, increase toward `10 * k` if recall is short.
|
||||
- IVF candidate breadth: `nprobes(...)` is usually auto-tuned; override only when a selective pre-filter leaves too few neighbors.
|
||||
|
||||
## Maintenance
|
||||
|
||||
After every successful embedded OSS/local ingestion, call `table.optimize()`.
|
||||
Do not add this to LanceDB Enterprise/Cloud remote table code; remote compaction
|
||||
and cleanup are handled automatically based on the Enterprise cluster
|
||||
configuration.
|
||||
|
||||
Why local maintenance is needed:
|
||||
|
||||
- Frequent writes can create many small fragments. Queries then need to scan across more files, which can increase latency.
|
||||
- Updates, deletes, and appends create new table versions. Old versions are retained for time travel and rollback, which can grow disk usage.
|
||||
- Indexes may have newly added rows that are not yet fully optimized into the index structure.
|
||||
|
||||
For local/OSS tables, run `optimize()` after the final successful ingestion
|
||||
write. Also run it after later batches of update/delete operations or on a
|
||||
regular maintenance schedule:
|
||||
|
||||
```typescript
|
||||
await table.optimize();
|
||||
```
|
||||
|
||||
If the user wants more aggressive local disk cleanup, pass a shorter cleanup retention window:
|
||||
|
||||
```typescript
|
||||
const olderThan = new Date(Date.now() - 24 * 60 * 60 * 1000);
|
||||
await table.optimize({ cleanupOlderThan: olderThan });
|
||||
```
|
||||
|
||||
Do not use very short cleanup windows when the application depends on time travel, rollback, or old versions.
|
||||
|
||||
## Diagnostics
|
||||
|
||||
Before changing code or indexes, inspect:
|
||||
|
||||
```typescript
|
||||
console.log(await table.search(queryVector).where("year > 2000").limit(10).analyzePlan());
|
||||
console.log(await table.indexStats("vector_idx"));
|
||||
```
|
||||
|
||||
Look for high scan cost, missing indexes, fragmented data, and unindexed rows.
|
||||
+9
-9
@@ -15,10 +15,10 @@ name = "_lancedb"
|
||||
crate-type = ["cdylib"]
|
||||
|
||||
[dependencies]
|
||||
arrow = { version = "58.0.0", features = ["pyarrow"] }
|
||||
async-trait = "0.1"
|
||||
bytes = "1"
|
||||
lancedb = { path = "../rust/lancedb", default-features = false }
|
||||
arrow = { workspace = true, features = ["pyarrow"] }
|
||||
async-trait.workspace = true
|
||||
bytes.workspace = true
|
||||
lancedb.workspace = true
|
||||
datafusion-common.workspace = true
|
||||
lance-core.workspace = true
|
||||
lance-namespace.workspace = true
|
||||
@@ -27,17 +27,17 @@ lance-io.workspace = true
|
||||
env_logger.workspace = true
|
||||
log.workspace = true
|
||||
pyo3 = { version = "0.28", features = ["extension-module", "abi3-py310", "chrono"] }
|
||||
chrono = { version = "0.4", default-features = false, features = ["clock"] }
|
||||
chrono.workspace = true
|
||||
pyo3-async-runtimes = { version = "0.28", features = [
|
||||
"attributes",
|
||||
"tokio-runtime",
|
||||
] }
|
||||
pin-project = "1.1.5"
|
||||
pin-project.workspace = true
|
||||
futures.workspace = true
|
||||
serde = "1"
|
||||
serde_json = "1"
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
snafu.workspace = true
|
||||
tokio = { version = "1.40", features = ["sync", "rt-multi-thread"] }
|
||||
tokio.workspace = true
|
||||
libc = "0.2"
|
||||
|
||||
[build-dependencies]
|
||||
|
||||
@@ -22,6 +22,12 @@ from .remote.db import RemoteDBConnection
|
||||
from .expr import Expr, col, lit, func
|
||||
from .schema import blob, vector, BlobType
|
||||
from .job import AsyncJob, Job
|
||||
from .functions import (
|
||||
FunctionApplication as FunctionApplication,
|
||||
FunctionBinding as FunctionBinding,
|
||||
FunctionVersion as FunctionVersion,
|
||||
PythonRuntimeSpec as PythonRuntimeSpec,
|
||||
)
|
||||
from .table import AsyncTable, CompactionOptions, Table
|
||||
from .types import BaseTokenizerType
|
||||
from ._lancedb import Session
|
||||
|
||||
@@ -0,0 +1,379 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
"""Canonical values exchanged with LanceDB Enterprise Function services.
|
||||
|
||||
These immutable models contain client/wire state only. Catalog persistence,
|
||||
environment bake, secret resolution, and execution are owned by Sophon.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, Optional
|
||||
|
||||
import pydantic
|
||||
from pydantic import BaseModel, Field, conint
|
||||
|
||||
_PYDANTIC_V2 = int(pydantic.VERSION.split(".", 1)[0]) >= 2
|
||||
if _PYDANTIC_V2:
|
||||
from pydantic import field_validator, model_validator
|
||||
else:
|
||||
from pydantic import root_validator, validator
|
||||
|
||||
_Int32 = conint(strict=True, ge=-(2**31), le=2**31 - 1)
|
||||
_UInt32 = conint(strict=True, ge=0, le=2**32 - 1)
|
||||
_UInt64 = conint(strict=True, ge=0, le=2**64 - 1)
|
||||
|
||||
|
||||
class _FrozenDict(dict):
|
||||
def _immutable(self, *args, **kwargs):
|
||||
raise TypeError("remote canonical values are immutable")
|
||||
|
||||
__setitem__ = _immutable
|
||||
__delitem__ = _immutable
|
||||
clear = _immutable
|
||||
pop = _immutable
|
||||
popitem = _immutable
|
||||
setdefault = _immutable
|
||||
update = _immutable
|
||||
|
||||
def __ior__(self, other):
|
||||
self._immutable()
|
||||
|
||||
|
||||
def _freeze_value(value):
|
||||
if isinstance(value, Mapping):
|
||||
return _FrozenDict({key: _freeze_value(child) for key, child in value.items()})
|
||||
if isinstance(value, (list, tuple)):
|
||||
return tuple(_freeze_value(child) for child in value)
|
||||
return value
|
||||
|
||||
|
||||
def _validate_literal(value):
|
||||
if isinstance(value, float):
|
||||
raise ValueError(
|
||||
"floating-point Function literals are not part of the Slice 1 "
|
||||
"canonical wire contract"
|
||||
)
|
||||
if isinstance(value, int) and not isinstance(value, bool):
|
||||
if not -(2**63) <= value <= 2**64 - 1:
|
||||
raise ValueError(
|
||||
"Function integer literal is outside the canonical JSON range"
|
||||
)
|
||||
elif isinstance(value, Mapping):
|
||||
for child in value.values():
|
||||
_validate_literal(child)
|
||||
elif isinstance(value, (list, tuple)):
|
||||
for child in value:
|
||||
_validate_literal(child)
|
||||
return value
|
||||
|
||||
|
||||
def _known_wire_value(value):
|
||||
if isinstance(value, _RemoteValue):
|
||||
return value._known_dict()
|
||||
if isinstance(value, Mapping):
|
||||
return {key: _known_wire_value(child) for key, child in value.items()}
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [_known_wire_value(child) for child in value]
|
||||
return value
|
||||
|
||||
|
||||
class _RemoteValue(BaseModel):
|
||||
if _PYDANTIC_V2:
|
||||
model_config = {"extra": "ignore", "frozen": True}
|
||||
else:
|
||||
|
||||
class Config:
|
||||
allow_mutation = False
|
||||
extra = "ignore"
|
||||
|
||||
if _PYDANTIC_V2:
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _freeze_mappings(self):
|
||||
for name, value in self.__dict__.items():
|
||||
object.__setattr__(self, name, _freeze_value(value))
|
||||
return self
|
||||
|
||||
else:
|
||||
|
||||
@root_validator
|
||||
def _freeze_mappings(cls, values):
|
||||
return {name: _freeze_value(value) for name, value in values.items()}
|
||||
|
||||
@classmethod
|
||||
def from_json(cls, payload: str):
|
||||
if _PYDANTIC_V2:
|
||||
return cls.model_validate_json(payload)
|
||||
return cls.parse_raw(payload)
|
||||
|
||||
def _known_dict(self) -> dict[str, Any]:
|
||||
fields = self.__class__.model_fields if _PYDANTIC_V2 else self.__fields__
|
||||
known = {}
|
||||
for name, field in fields.items():
|
||||
value = getattr(self, name)
|
||||
if value is None:
|
||||
continue
|
||||
required = field.is_required() if _PYDANTIC_V2 else field.required
|
||||
if not required:
|
||||
default_factory = field.default_factory
|
||||
if default_factory is not None and value == default_factory():
|
||||
continue
|
||||
if default_factory is None and value == field.default:
|
||||
continue
|
||||
known[name] = _known_wire_value(value)
|
||||
return known
|
||||
|
||||
def _copy(self, *, update: Mapping[str, Any]):
|
||||
update = {name: _freeze_value(value) for name, value in update.items()}
|
||||
if _PYDANTIC_V2:
|
||||
return self.model_copy(update=update)
|
||||
return self.copy(update=update)
|
||||
|
||||
def to_canonical_json(self) -> str:
|
||||
return json.dumps(
|
||||
self._known_dict(),
|
||||
ensure_ascii=False,
|
||||
allow_nan=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
|
||||
|
||||
class FunctionArtifact(_RemoteValue):
|
||||
"""Content-addressed Python artifact identity."""
|
||||
|
||||
kind: str
|
||||
digest: str
|
||||
entrypoint: str
|
||||
|
||||
|
||||
class FunctionParameter(_RemoteValue):
|
||||
name: str
|
||||
arrow_type: str
|
||||
nullable: bool
|
||||
|
||||
|
||||
class FunctionResultField(_RemoteValue):
|
||||
name: str
|
||||
arrow_type: str
|
||||
nullable: bool
|
||||
|
||||
|
||||
class FunctionOutput(_RemoteValue):
|
||||
"""Scalar or ordered named-struct output; unknown kinds remain decodable."""
|
||||
|
||||
kind: str
|
||||
arrow_type: Optional[str] = None
|
||||
nullable: Optional[bool] = None
|
||||
fields: tuple[FunctionResultField, ...] = ()
|
||||
|
||||
|
||||
class FunctionSignature(_RemoteValue):
|
||||
inputs: tuple[FunctionParameter, ...]
|
||||
output: FunctionOutput
|
||||
|
||||
|
||||
class PythonEnvironmentSpec(_RemoteValue):
|
||||
"""One Sophon-managed Python environment source."""
|
||||
|
||||
kind: str
|
||||
packages: tuple[str, ...] = ()
|
||||
path: Optional[str] = None
|
||||
modules: tuple[str, ...] = ()
|
||||
image: Optional[str] = None
|
||||
|
||||
|
||||
class PythonRuntimeSpec(_RemoteValue):
|
||||
"""Remote runtime definition with non-secret environment values.
|
||||
|
||||
V1 supports ``kind="python"``. Newer runtime kinds remain readable, while
|
||||
their unknown payload fields are intentionally not retained by the client.
|
||||
"""
|
||||
|
||||
kind: str
|
||||
python_version: Optional[str] = None
|
||||
environment: Optional[PythonEnvironmentSpec] = None
|
||||
env: Optional[Mapping[str, str]] = None
|
||||
|
||||
if _PYDANTIC_V2:
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_runtime_kind(self):
|
||||
if self.kind == "python":
|
||||
if self.python_version is None:
|
||||
raise ValueError("python runtime requires python_version")
|
||||
if self.environment is None:
|
||||
raise ValueError("python runtime requires environment")
|
||||
else:
|
||||
object.__setattr__(self, "python_version", None)
|
||||
object.__setattr__(self, "environment", None)
|
||||
object.__setattr__(self, "env", None)
|
||||
return self
|
||||
|
||||
else:
|
||||
|
||||
@root_validator
|
||||
def _validate_runtime_kind(cls, values):
|
||||
if values.get("kind") == "python":
|
||||
if values.get("python_version") is None:
|
||||
raise ValueError("python runtime requires python_version")
|
||||
if values.get("environment") is None:
|
||||
raise ValueError("python runtime requires environment")
|
||||
else:
|
||||
values["python_version"] = None
|
||||
values["environment"] = None
|
||||
values["env"] = None
|
||||
return values
|
||||
|
||||
|
||||
class FunctionVersion(_RemoteValue):
|
||||
"""An exact immutable Function version returned by Enterprise.
|
||||
|
||||
Scheduling resources, priority, concurrency, and retry policy belong to
|
||||
the submitting Job and are not part of this identity.
|
||||
"""
|
||||
|
||||
name: str
|
||||
version: str
|
||||
artifact: FunctionArtifact
|
||||
signature: FunctionSignature
|
||||
runtime: PythonRuntimeSpec
|
||||
runtime_digest: str
|
||||
environment_digest: str
|
||||
required_secrets: tuple[str, ...] = ()
|
||||
created_at: str
|
||||
|
||||
|
||||
class FunctionVersionRef(_RemoteValue):
|
||||
name: str
|
||||
version: str
|
||||
|
||||
|
||||
class ApplicationInput(_RemoteValue):
|
||||
"""One parameter value.
|
||||
|
||||
Slice 1 freezes integers, strings, booleans, nulls, arrays, and objects.
|
||||
Floating-point literal encoding is deferred until Python authoring is
|
||||
introduced with a language-neutral numeric representation.
|
||||
"""
|
||||
|
||||
parameter: str
|
||||
kind: str
|
||||
value: Any
|
||||
|
||||
if _PYDANTIC_V2:
|
||||
|
||||
@field_validator("value")
|
||||
@classmethod
|
||||
def _validate_value(cls, value):
|
||||
return _validate_literal(value)
|
||||
|
||||
else:
|
||||
|
||||
@validator("value")
|
||||
def _validate_value(cls, value):
|
||||
return _validate_literal(value)
|
||||
|
||||
|
||||
class FunctionApplication(_RemoteValue):
|
||||
"""Immutable pre-declaration application of an exact Function version."""
|
||||
|
||||
function: FunctionVersionRef
|
||||
inputs: tuple[ApplicationInput, ...]
|
||||
output: FunctionOutput
|
||||
group_id: str
|
||||
columns: Mapping[str, str] = Field(default_factory=dict)
|
||||
|
||||
def rename(self, *, columns: Mapping[str, str]) -> FunctionApplication:
|
||||
"""Return a copy with result-field to table-column aliases."""
|
||||
if self.output.kind != "named_struct":
|
||||
raise ValueError("rename(columns=...) requires a named-struct application")
|
||||
result_fields = {field.name for field in self.output.fields}
|
||||
unknown = set(columns) - result_fields
|
||||
if unknown:
|
||||
raise ValueError(f"unknown Function result fields: {sorted(unknown)!r}")
|
||||
merged = dict(self.columns)
|
||||
merged.update(columns)
|
||||
destinations = tuple(
|
||||
merged.get(field.name, field.name) for field in self.output.fields
|
||||
)
|
||||
if len(set(destinations)) != len(destinations):
|
||||
raise ValueError("FunctionApplication rename destinations must be unique")
|
||||
return self._copy(update={"columns": merged})
|
||||
|
||||
|
||||
class InputBinding(_RemoteValue):
|
||||
parameter: str
|
||||
field_id: _Int32
|
||||
field_path: str
|
||||
arrow_type: str
|
||||
nullable: bool
|
||||
|
||||
|
||||
class OutputMapping(_RemoteValue):
|
||||
"""One stable result-field mapping.
|
||||
|
||||
Assignment state is outside the Slice 1 client contract. During the NULL
|
||||
transition Lance exposes no public cell-flag identifier to persist here.
|
||||
"""
|
||||
|
||||
result_field: str
|
||||
output_name: str
|
||||
output_field_id: _Int32
|
||||
output_ordinal: _UInt32
|
||||
arrow_type: str
|
||||
nullable: bool
|
||||
|
||||
|
||||
class FunctionBinding(_RemoteValue):
|
||||
"""Immutable grouped binding persisted by the Enterprise table service."""
|
||||
|
||||
binding_id: str
|
||||
revision: _UInt64
|
||||
function: FunctionVersionRef
|
||||
group_id: str
|
||||
inputs: tuple[InputBinding, ...]
|
||||
outputs: tuple[OutputMapping, ...]
|
||||
|
||||
|
||||
class RefreshColumnResult(_RemoteValue):
|
||||
"""Terminal result of a remote Function-column refresh Job."""
|
||||
|
||||
rows_assigned: _UInt64
|
||||
rows_failed: _UInt64
|
||||
rows_remaining: _UInt64
|
||||
source_version: _UInt64
|
||||
published_version: Optional[_UInt64] = None
|
||||
|
||||
@property
|
||||
def rows_filled(self) -> int:
|
||||
"""Deprecated compatibility alias for :attr:`rows_assigned`."""
|
||||
return self.rows_assigned
|
||||
|
||||
@property
|
||||
def version(self) -> Optional[int]:
|
||||
"""Deprecated compatibility alias for :attr:`published_version`."""
|
||||
return self.published_version
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ApplicationInput",
|
||||
"FunctionApplication",
|
||||
"FunctionArtifact",
|
||||
"FunctionBinding",
|
||||
"FunctionOutput",
|
||||
"FunctionParameter",
|
||||
"FunctionResultField",
|
||||
"FunctionSignature",
|
||||
"FunctionVersion",
|
||||
"FunctionVersionRef",
|
||||
"InputBinding",
|
||||
"OutputMapping",
|
||||
"PythonEnvironmentSpec",
|
||||
"PythonRuntimeSpec",
|
||||
"RefreshColumnResult",
|
||||
]
|
||||
@@ -516,6 +516,20 @@ def _cast_to_target_schema(
|
||||
return pa.RecordBatchReader.from_batches(reordered_schema, gen())
|
||||
|
||||
|
||||
def _field_extension_name(field: pa.Field) -> Optional[str]:
|
||||
extension_name = getattr(field.type, "extension_name", None)
|
||||
if extension_name is not None:
|
||||
return extension_name
|
||||
|
||||
metadata = field.metadata or {}
|
||||
extension_name = metadata.get(b"ARROW:extension:name") or metadata.get(
|
||||
"ARROW:extension:name"
|
||||
)
|
||||
if isinstance(extension_name, bytes):
|
||||
return extension_name.decode()
|
||||
return extension_name
|
||||
|
||||
|
||||
def _align_field_types(
|
||||
fields: List[pa.Field],
|
||||
target_fields: List[pa.Field],
|
||||
@@ -528,6 +542,16 @@ def _align_field_types(
|
||||
target_field = next((f for f in target_fields if f.name == field.name), None)
|
||||
if target_field is None:
|
||||
raise ValueError(f"Field '{field.name}' not found in target schema")
|
||||
# Preserve arrow.json input until it reaches Lance. LanceDB exposes stored
|
||||
# JSON columns as lance.json (JSONB-backed LargeBinary), but casting the
|
||||
# input to that storage type here merely relabels the raw JSON bytes as
|
||||
# JSONB. Lance must see arrow.json so it can perform the JSONB encoding.
|
||||
if (
|
||||
_field_extension_name(field) == "arrow.json"
|
||||
and _field_extension_name(target_field) == "lance.json"
|
||||
):
|
||||
new_fields.append(field)
|
||||
continue
|
||||
if pa.types.is_struct(target_field.type):
|
||||
if pa.types.is_struct(field.type):
|
||||
new_type = pa.struct(
|
||||
|
||||
@@ -0,0 +1,225 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import lancedb.functions as functions
|
||||
from lancedb.functions import (
|
||||
FunctionApplication,
|
||||
FunctionBinding,
|
||||
FunctionVersion,
|
||||
PythonRuntimeSpec,
|
||||
RefreshColumnResult,
|
||||
)
|
||||
|
||||
|
||||
FIXTURES = (
|
||||
Path(__file__).parents[3]
|
||||
/ "rust"
|
||||
/ "lancedb"
|
||||
/ "tests"
|
||||
/ "fixtures"
|
||||
/ "first_class_functions"
|
||||
/ "v1"
|
||||
)
|
||||
|
||||
|
||||
def fixture(name: str) -> str:
|
||||
return (FIXTURES / name).read_text()
|
||||
|
||||
|
||||
def job_result(name: str) -> dict:
|
||||
return json.loads(fixture(name))["result"]
|
||||
|
||||
|
||||
def assert_no_secret_values(value):
|
||||
if isinstance(value, dict):
|
||||
for key, child in value.items():
|
||||
assert key not in {
|
||||
"secret_value",
|
||||
"secret_values",
|
||||
"resolved_secret",
|
||||
"resolved_secrets",
|
||||
}
|
||||
assert_no_secret_values(child)
|
||||
elif isinstance(value, list):
|
||||
for child in value:
|
||||
assert_no_secret_values(child)
|
||||
|
||||
|
||||
def test_public_function_values_are_in_api_reference():
|
||||
docs = Path(__file__).parents[3] / "docs" / "src" / "python" / "python.md"
|
||||
rendered = docs.read_text()
|
||||
for name in functions.__all__:
|
||||
assert f"::: lancedb.functions.{name}" in rendered
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("fixture_name", "canonical_name", "model", "nested_result"),
|
||||
[
|
||||
(
|
||||
"remote_function_job.json",
|
||||
"remote_function_version.canonical.json",
|
||||
FunctionVersion,
|
||||
True,
|
||||
),
|
||||
(
|
||||
"remote_function_application.json",
|
||||
"remote_function_application.canonical.json",
|
||||
FunctionApplication,
|
||||
False,
|
||||
),
|
||||
(
|
||||
"remote_function_binding.json",
|
||||
"remote_function_binding.canonical.json",
|
||||
FunctionBinding,
|
||||
False,
|
||||
),
|
||||
(
|
||||
"remote_refresh_job.json",
|
||||
"remote_refresh_result.canonical.json",
|
||||
RefreshColumnResult,
|
||||
True,
|
||||
),
|
||||
(
|
||||
"remote_refresh_result_without_published_version.json",
|
||||
"remote_refresh_result_without_published_version.canonical.json",
|
||||
RefreshColumnResult,
|
||||
False,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_python_and_rust_share_remote_canonical_goldens(
|
||||
fixture_name, canonical_name, model, nested_result
|
||||
):
|
||||
value = json.loads(fixture(fixture_name))
|
||||
if nested_result:
|
||||
value = value["result"]
|
||||
decoded = model.from_json(json.dumps(value))
|
||||
assert decoded.to_canonical_json() == fixture(canonical_name).strip()
|
||||
|
||||
|
||||
def test_function_version_identity_is_immutable_and_exact():
|
||||
value = job_result("remote_function_job.json")
|
||||
version = FunctionVersion.from_json(json.dumps(value))
|
||||
assert version.name == "embed"
|
||||
assert version.version == "fv_01K3EXACT"
|
||||
assert version.required_secrets == ("HF_TOKEN",)
|
||||
|
||||
with pytest.raises((TypeError, ValueError)):
|
||||
version.version = "fv_changed"
|
||||
with pytest.raises(TypeError, match="immutable"):
|
||||
version.runtime.env["TOKENIZERS_PARALLELISM"] = "true"
|
||||
|
||||
changed = dict(value)
|
||||
changed["version"] = "fv_changed"
|
||||
assert FunctionVersion(**changed) != version
|
||||
|
||||
|
||||
def test_unknown_fields_and_discriminators_are_forward_decodable():
|
||||
value = job_result("remote_function_job.json")
|
||||
value["future_version_metadata"] = {"retention_class": "catalog"}
|
||||
value["runtime"] = {"kind": "wasm", "module_digest": "sha256:wasm"}
|
||||
value["signature"]["output"]["kind"] = "future_output_shape"
|
||||
|
||||
version = FunctionVersion.from_json(json.dumps(value))
|
||||
assert version.runtime.kind == "wasm"
|
||||
assert version.runtime.python_version is None
|
||||
assert version.runtime.environment is None
|
||||
assert json.loads(version.to_canonical_json())["runtime"] == {"kind": "wasm"}
|
||||
assert version.signature.output.kind == "future_output_shape"
|
||||
|
||||
|
||||
def test_function_application_uses_rename_columns_only():
|
||||
application = FunctionApplication.from_json(
|
||||
fixture("remote_function_application.json")
|
||||
)
|
||||
renamed = application.rename(columns={"normalized_text": "body_normalized"})
|
||||
|
||||
assert application.columns["normalized_text"] == "search_text"
|
||||
assert renamed.columns["normalized_text"] == "body_normalized"
|
||||
assert renamed.function == application.function
|
||||
assert renamed.group_id == application.group_id
|
||||
assert not hasattr(application, "rename_outputs")
|
||||
with pytest.raises(TypeError, match="immutable"):
|
||||
renamed.columns["normalized_text"] = "changed"
|
||||
with pytest.raises(TypeError, match="immutable"):
|
||||
application.inputs[0].value["path"] = "changed"
|
||||
|
||||
with pytest.raises(ValueError, match="unknown Function result fields"):
|
||||
application.rename(columns={"missing": "search_text"})
|
||||
with pytest.raises(ValueError, match="destinations must be unique"):
|
||||
application.rename(columns={"normalized_text": "same", "token_count": "same"})
|
||||
|
||||
bare_value = json.loads(fixture("remote_function_application.json"))
|
||||
bare_value.pop("columns")
|
||||
bare = FunctionApplication(**bare_value)
|
||||
with pytest.raises(ValueError, match="destinations must be unique"):
|
||||
bare.rename(columns={"normalized_text": "token_count"})
|
||||
|
||||
|
||||
def test_binding_and_refresh_result_keep_stable_remote_fields():
|
||||
binding = FunctionBinding.from_json(fixture("remote_function_binding.json"))
|
||||
assert binding.revision == 3
|
||||
assert binding.function.version == "fv_01K3TEXT"
|
||||
assert [output.output_ordinal for output in binding.outputs] == [0, 1]
|
||||
|
||||
result = RefreshColumnResult.from_json(
|
||||
json.dumps(job_result("remote_refresh_job.json"))
|
||||
)
|
||||
assert result.rows_filled == result.rows_assigned
|
||||
assert result.version == result.published_version
|
||||
|
||||
result = RefreshColumnResult.from_json(
|
||||
fixture("remote_refresh_result_without_published_version.json")
|
||||
)
|
||||
assert result.published_version is None
|
||||
assert RefreshColumnResult.from_json(result.to_canonical_json()) == result
|
||||
|
||||
|
||||
def test_function_literal_numeric_domain_matches_rust():
|
||||
with pytest.raises(ValueError, match="floating-point Function literals"):
|
||||
FunctionApplication.from_json(fixture("remote_function_application_float.json"))
|
||||
|
||||
value = json.loads(fixture("remote_function_application_float.json"))
|
||||
value["inputs"][0]["value"] = 2**64
|
||||
with pytest.raises(ValueError, match="outside the canonical JSON range"):
|
||||
FunctionApplication.from_json(json.dumps(value))
|
||||
|
||||
|
||||
def test_empty_default_maps_have_stable_canonical_bytes():
|
||||
runtime = PythonRuntimeSpec(
|
||||
kind="python", python_version="3.12", environment={"kind": "pip"}
|
||||
)
|
||||
assert runtime.to_canonical_json() == (
|
||||
'{"environment":{"kind":"pip"},"kind":"python","python_version":"3.12"}'
|
||||
)
|
||||
|
||||
value = json.loads(fixture("remote_function_application.json"))
|
||||
value.pop("columns")
|
||||
application = FunctionApplication.from_json(json.dumps(value))
|
||||
assert "columns" not in json.loads(application.to_canonical_json())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ["rows_assigned", "source_version"])
|
||||
def test_refresh_result_rejects_non_u64_values(field):
|
||||
value = job_result("remote_refresh_job.json")
|
||||
value[field] = -1
|
||||
with pytest.raises(ValueError):
|
||||
RefreshColumnResult.from_json(json.dumps(value))
|
||||
|
||||
value[field] = "1"
|
||||
with pytest.raises(ValueError):
|
||||
RefreshColumnResult.from_json(json.dumps(value))
|
||||
|
||||
|
||||
def test_canonical_client_values_contain_secret_names_only():
|
||||
version = FunctionVersion.from_json(
|
||||
json.dumps(job_result("remote_function_job.json"))
|
||||
)
|
||||
canonical = json.loads(version.to_canonical_json())
|
||||
assert canonical["required_secrets"] == ["HF_TOKEN"]
|
||||
assert_no_secret_values(canonical)
|
||||
@@ -2772,6 +2772,56 @@ async def test_merge_insert_async(mem_db_async: AsyncConnection):
|
||||
assert (await table.to_arrow()).sort_by("a") == expected
|
||||
|
||||
|
||||
@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type")
|
||||
@pytest.mark.asyncio
|
||||
async def test_merge_insert_encodes_json(mem_db_async: AsyncConnection):
|
||||
json_type = pa.json_()
|
||||
schema = pa.schema([pa.field("id", pa.string()), pa.field("j", json_type)])
|
||||
|
||||
def json_table(rows):
|
||||
json_values = pa.ExtensionArray.from_storage(
|
||||
json_type,
|
||||
pa.array([value for _, value in rows], type=json_type.storage_type),
|
||||
)
|
||||
return pa.Table.from_arrays(
|
||||
[pa.array([row_id for row_id, _ in rows]), json_values], schema=schema
|
||||
)
|
||||
|
||||
table = await mem_db_async.create_table("json_merge", schema=schema)
|
||||
await table.add(json_table([("a", '{"k": 1}'), ("b", '{"k": 9}')]))
|
||||
|
||||
await (
|
||||
table.merge_insert("id")
|
||||
.when_matched_update_all()
|
||||
.execute(json_table([("a", '{"k": 2}')]))
|
||||
)
|
||||
|
||||
rows = sorted(await table.query().to_list(), key=lambda row: row["id"])
|
||||
assert rows == [
|
||||
{"id": "a", "j": '{"k":2}'},
|
||||
{"id": "b", "j": '{"k":9}'},
|
||||
]
|
||||
filtered = await table.query().where("json_extract(j, '$.k') = '2'").to_list()
|
||||
assert filtered == [{"id": "a", "j": '{"k":2}'}]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type")
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_sanitization_encodes_json(mem_db_async: AsyncConnection):
|
||||
json_type = pa.json_()
|
||||
schema = pa.schema([pa.field("id", pa.string()), pa.field("j", json_type)])
|
||||
json_values = pa.ExtensionArray.from_storage(
|
||||
json_type, pa.array(['{"k": 3}'], type=json_type.storage_type)
|
||||
)
|
||||
data = pa.Table.from_arrays([pa.array(["c"]), json_values], schema=schema)
|
||||
|
||||
table = await mem_db_async.create_table("json_add", schema=schema)
|
||||
await table.add(data, on_bad_vectors="fill")
|
||||
|
||||
rows = await table.query().where("json_extract(j, '$.k') = '3'").to_list()
|
||||
assert rows == [{"id": "c", "j": '{"k":3}'}]
|
||||
|
||||
|
||||
def test_create_with_embedding_function(mem_db: DBConnection):
|
||||
class MyTable(LanceModel):
|
||||
text: str
|
||||
|
||||
+10
-10
@@ -51,20 +51,20 @@ metrics = { workspace = true, optional = true }
|
||||
metrics-util = { workspace = true, optional = true }
|
||||
moka = { workspace = true }
|
||||
pin-project = { workspace = true }
|
||||
tokio = { version = "1.23", features = ["rt-multi-thread", "sync"] }
|
||||
tokio = { workspace = true }
|
||||
log.workspace = true
|
||||
async-trait = "0"
|
||||
bytes = "1"
|
||||
async-trait = { workspace = true }
|
||||
bytes = { workspace = true }
|
||||
futures.workspace = true
|
||||
num-traits.workspace = true
|
||||
url.workspace = true
|
||||
rand.workspace = true
|
||||
regex.workspace = true
|
||||
serde = { version = "^1" }
|
||||
serde_json = { version = "1" }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
async-openai = { version = "0.20.0", optional = true }
|
||||
serde_with = { version = "3.8.1" }
|
||||
tempfile = "3.5.0"
|
||||
tempfile = { workspace = true }
|
||||
aws-sdk-bedrockruntime = { version = "1.27.0", optional = true }
|
||||
# For remote feature
|
||||
reqwest = { version = "0.12.0", default-features = false, features = [
|
||||
@@ -79,7 +79,7 @@ reqwest = { version = "0.12.0", default-features = false, features = [
|
||||
], optional = true }
|
||||
http = { version = "1", optional = true } # Matching what is in reqwest
|
||||
urlencoding = { version = "2", optional = true }
|
||||
uuid = { version = "1.7.0", features = ["v4", "v5"] }
|
||||
uuid = { workspace = true, features = ["v5"] }
|
||||
polars-arrow = { version = ">=0.37,<0.40.0", optional = true }
|
||||
polars = { version = ">=0.37,<0.40.0", optional = true }
|
||||
hf-hub = { version = "0.4.1", optional = true, default-features = false, features = [
|
||||
@@ -96,11 +96,11 @@ semver = { workspace = true }
|
||||
[dev-dependencies]
|
||||
anyhow = "1"
|
||||
lance-testing = { workspace = true }
|
||||
tempfile = "3.5.0"
|
||||
tempfile = { workspace = true }
|
||||
random_word = { version = "0.4.3", features = ["en"] }
|
||||
roaring = "0.11.4"
|
||||
tokio = { version = "1.23", features = ["io-util", "macros", "net", "rt-multi-thread", "sync", "test-util"] }
|
||||
uuid = { version = "1.7.0", features = ["v4"] }
|
||||
tokio = { workspace = true, features = ["io-util", "macros", "net", "test-util"] }
|
||||
uuid = { workspace = true }
|
||||
walkdir = "2"
|
||||
aws-sdk-dynamodb = { version = "1.55.0" }
|
||||
aws-sdk-s3 = { version = "1.55.0" }
|
||||
|
||||
@@ -0,0 +1,489 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
//! Canonical values exchanged with the Enterprise Function service.
|
||||
//!
|
||||
//! This module contains client/wire values only. Catalog persistence,
|
||||
//! environment bake, secret resolution, and execution are owned by Sophon.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use serde::de::{self, DeserializeOwned};
|
||||
use serde::{Deserialize, Deserializer, Serialize, Serializer};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{Error, Result};
|
||||
|
||||
fn invalid_json(error: impl std::fmt::Display) -> Error {
|
||||
Error::InvalidInput {
|
||||
message: format!("invalid remote Function JSON: {error}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn write_canonical_json(value: &Value, output: &mut String) -> serde_json::Result<()> {
|
||||
match value {
|
||||
Value::Object(map) => {
|
||||
output.push('{');
|
||||
let mut entries = map.iter().collect::<Vec<_>>();
|
||||
entries.sort_unstable_by_key(|(key, _)| *key);
|
||||
for (index, (key, value)) in entries.into_iter().enumerate() {
|
||||
if index != 0 {
|
||||
output.push(',');
|
||||
}
|
||||
output.push_str(&serde_json::to_string(key)?);
|
||||
output.push(':');
|
||||
write_canonical_json(value, output)?;
|
||||
}
|
||||
output.push('}');
|
||||
}
|
||||
Value::Array(values) => {
|
||||
output.push('[');
|
||||
for (index, value) in values.iter().enumerate() {
|
||||
if index != 0 {
|
||||
output.push(',');
|
||||
}
|
||||
write_canonical_json(value, output)?;
|
||||
}
|
||||
output.push(']');
|
||||
}
|
||||
other => output.push_str(&serde_json::to_string(other)?),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn canonical_json<T: Serialize>(value: &T) -> Result<String> {
|
||||
let value = serde_json::to_value(value).map_err(invalid_json)?;
|
||||
let mut output = String::new();
|
||||
write_canonical_json(&value, &mut output).map_err(invalid_json)?;
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
fn from_json<T: DeserializeOwned>(json: &str) -> Result<T> {
|
||||
serde_json::from_str(json).map_err(invalid_json)
|
||||
}
|
||||
|
||||
fn validate_literal(value: &Value) -> Result<()> {
|
||||
match value {
|
||||
Value::Number(number) if number.is_f64() => Err(Error::InvalidInput {
|
||||
message: "floating-point Function literals are not part of the Slice 1 canonical wire contract"
|
||||
.to_string(),
|
||||
}),
|
||||
Value::Array(values) => values.iter().try_for_each(validate_literal),
|
||||
Value::Object(values) => values.values().try_for_each(validate_literal),
|
||||
_ => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
macro_rules! impl_json {
|
||||
($type:ty) => {
|
||||
impl $type {
|
||||
/// Decode a remote value. Unknown fields and discriminator values
|
||||
/// are accepted so newer servers remain readable.
|
||||
pub fn from_json(json: &str) -> Result<Self> {
|
||||
from_json(json)
|
||||
}
|
||||
|
||||
/// Encode the known client contract with bytewise-sorted JSON keys.
|
||||
pub fn to_canonical_json(&self) -> Result<String> {
|
||||
canonical_json(self)
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/// Packaged Python artifact identity. Source bytes are never part of this value.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct FunctionArtifact {
|
||||
pub kind: String,
|
||||
pub digest: String,
|
||||
pub entrypoint: String,
|
||||
}
|
||||
|
||||
/// One ordered Arrow input parameter.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct FunctionParameter {
|
||||
pub name: String,
|
||||
pub arrow_type: String,
|
||||
pub nullable: bool,
|
||||
}
|
||||
|
||||
/// One field of an ordered named-struct result.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct FunctionResultField {
|
||||
pub name: String,
|
||||
pub arrow_type: String,
|
||||
pub nullable: bool,
|
||||
}
|
||||
|
||||
/// Scalar or named-struct Function output.
|
||||
///
|
||||
/// `kind` remains a string so unknown future result shapes can be decoded.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct FunctionOutput {
|
||||
pub kind: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub arrow_type: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub nullable: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub fields: Vec<FunctionResultField>,
|
||||
}
|
||||
|
||||
/// Ordered language-neutral Function signature.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct FunctionSignature {
|
||||
pub inputs: Vec<FunctionParameter>,
|
||||
pub output: FunctionOutput,
|
||||
}
|
||||
|
||||
/// One Python environment source.
|
||||
///
|
||||
/// The selected source is interpreted by Sophon. `kind` is open for forward
|
||||
/// compatible decoding.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct PythonEnvironmentSpec {
|
||||
pub kind: String,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub packages: Vec<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub path: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub modules: Vec<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub image: Option<String>,
|
||||
}
|
||||
|
||||
/// Reproducible Python runtime definition understood by Sophon.
|
||||
///
|
||||
/// `env` contains non-secret values. Secret values have no client model;
|
||||
/// [`FunctionVersion::required_secrets`] contains names only.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[non_exhaustive]
|
||||
pub enum PythonRuntimeSpec {
|
||||
/// The V1 Sophon-managed Python runtime.
|
||||
Python {
|
||||
python_version: String,
|
||||
environment: PythonEnvironmentSpec,
|
||||
env: BTreeMap<String, String>,
|
||||
},
|
||||
/// A runtime kind introduced by a newer server.
|
||||
///
|
||||
/// Unknown payload fields are intentionally not retained because the
|
||||
/// client does not proxy catalog values.
|
||||
Unrecognized { kind: String },
|
||||
}
|
||||
|
||||
impl PythonRuntimeSpec {
|
||||
/// The wire discriminator reported by Sophon.
|
||||
pub fn kind(&self) -> &str {
|
||||
match self {
|
||||
Self::Python { .. } => "python",
|
||||
Self::Unrecognized { kind } => kind,
|
||||
}
|
||||
}
|
||||
|
||||
/// The Python version for the V1 runtime, or `None` for an unknown kind.
|
||||
pub fn python_version(&self) -> Option<&str> {
|
||||
match self {
|
||||
Self::Python { python_version, .. } => Some(python_version),
|
||||
Self::Unrecognized { .. } => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The Python environment for the V1 runtime, or `None` for an unknown kind.
|
||||
pub fn environment(&self) -> Option<&PythonEnvironmentSpec> {
|
||||
match self {
|
||||
Self::Python { environment, .. } => Some(environment),
|
||||
Self::Unrecognized { .. } => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Non-secret environment variables, or `None` for an unknown kind.
|
||||
pub fn env(&self) -> Option<&BTreeMap<String, String>> {
|
||||
match self {
|
||||
Self::Python { env, .. } => Some(env),
|
||||
Self::Unrecognized { .. } => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct PythonRuntimeWire {
|
||||
kind: String,
|
||||
#[serde(default)]
|
||||
python_version: Option<String>,
|
||||
#[serde(default)]
|
||||
environment: Option<PythonEnvironmentSpec>,
|
||||
#[serde(default)]
|
||||
env: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for PythonRuntimeSpec {
|
||||
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> std::result::Result<Self, D::Error> {
|
||||
let wire = PythonRuntimeWire::deserialize(deserializer)?;
|
||||
if wire.kind == "python" {
|
||||
Ok(Self::Python {
|
||||
python_version: wire
|
||||
.python_version
|
||||
.ok_or_else(|| de::Error::missing_field("python_version"))?,
|
||||
environment: wire
|
||||
.environment
|
||||
.ok_or_else(|| de::Error::missing_field("environment"))?,
|
||||
env: wire.env,
|
||||
})
|
||||
} else {
|
||||
Ok(Self::Unrecognized { kind: wire.kind })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Serialize for PythonRuntimeSpec {
|
||||
fn serialize<S: Serializer>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error> {
|
||||
#[derive(Serialize)]
|
||||
struct PythonRuntimeRef<'a> {
|
||||
kind: &'static str,
|
||||
python_version: &'a str,
|
||||
environment: &'a PythonEnvironmentSpec,
|
||||
#[serde(skip_serializing_if = "BTreeMap::is_empty")]
|
||||
env: &'a BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct UnrecognizedRuntimeRef<'a> {
|
||||
kind: &'a str,
|
||||
}
|
||||
|
||||
match self {
|
||||
Self::Python {
|
||||
python_version,
|
||||
environment,
|
||||
env,
|
||||
} => PythonRuntimeRef {
|
||||
kind: "python",
|
||||
python_version,
|
||||
environment,
|
||||
env,
|
||||
}
|
||||
.serialize(serializer),
|
||||
Self::Unrecognized { kind } => UnrecognizedRuntimeRef { kind }.serialize(serializer),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Immutable Function version returned by the Enterprise catalog.
|
||||
///
|
||||
/// Scheduling resources, priority, concurrency, and retry policy belong to
|
||||
/// the submitting Job and are not part of this identity.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct FunctionVersion {
|
||||
name: String,
|
||||
version: String,
|
||||
artifact: FunctionArtifact,
|
||||
signature: FunctionSignature,
|
||||
runtime: PythonRuntimeSpec,
|
||||
runtime_digest: String,
|
||||
environment_digest: String,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
required_secrets: Vec<String>,
|
||||
created_at: String,
|
||||
}
|
||||
|
||||
impl FunctionVersion {
|
||||
pub fn name(&self) -> &str {
|
||||
&self.name
|
||||
}
|
||||
|
||||
pub fn version(&self) -> &str {
|
||||
&self.version
|
||||
}
|
||||
|
||||
pub fn artifact(&self) -> &FunctionArtifact {
|
||||
&self.artifact
|
||||
}
|
||||
|
||||
pub fn signature(&self) -> &FunctionSignature {
|
||||
&self.signature
|
||||
}
|
||||
|
||||
pub fn runtime(&self) -> &PythonRuntimeSpec {
|
||||
&self.runtime
|
||||
}
|
||||
|
||||
pub fn runtime_digest(&self) -> &str {
|
||||
&self.runtime_digest
|
||||
}
|
||||
|
||||
pub fn environment_digest(&self) -> &str {
|
||||
&self.environment_digest
|
||||
}
|
||||
|
||||
/// Required secret names. Resolved values exist only inside Sophon.
|
||||
pub fn required_secrets(&self) -> &[String] {
|
||||
&self.required_secrets
|
||||
}
|
||||
|
||||
pub fn created_at(&self) -> &str {
|
||||
&self.created_at
|
||||
}
|
||||
}
|
||||
|
||||
impl_json!(FunctionVersion);
|
||||
|
||||
/// Exact FunctionVersion reference embedded in applications and bindings.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct FunctionVersionRef {
|
||||
pub name: String,
|
||||
pub version: String,
|
||||
}
|
||||
|
||||
/// Parameter binding in a FunctionApplication.
|
||||
///
|
||||
/// `kind` remains open until Python authoring is added in Slice 2. Slice 1
|
||||
/// freezes JSON integers, strings, booleans, nulls, arrays, and objects as
|
||||
/// canonical literal values. Floating-point literals are rejected until a
|
||||
/// language-neutral numeric representation is defined.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct ApplicationInput {
|
||||
pub parameter: String,
|
||||
pub kind: String,
|
||||
pub value: Value,
|
||||
}
|
||||
|
||||
/// Pre-declaration application of an exact FunctionVersion.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct FunctionApplication {
|
||||
function: FunctionVersionRef,
|
||||
inputs: Vec<ApplicationInput>,
|
||||
output: FunctionOutput,
|
||||
group_id: String,
|
||||
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||
columns: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
impl FunctionApplication {
|
||||
pub fn function(&self) -> &FunctionVersionRef {
|
||||
&self.function
|
||||
}
|
||||
|
||||
pub fn inputs(&self) -> &[ApplicationInput] {
|
||||
&self.inputs
|
||||
}
|
||||
|
||||
pub fn output(&self) -> &FunctionOutput {
|
||||
&self.output
|
||||
}
|
||||
|
||||
pub fn group_id(&self) -> &str {
|
||||
&self.group_id
|
||||
}
|
||||
|
||||
pub fn columns(&self) -> &BTreeMap<String, String> {
|
||||
&self.columns
|
||||
}
|
||||
|
||||
/// Decode a remote application after validating the Slice 1 literal domain.
|
||||
pub fn from_json(json: &str) -> Result<Self> {
|
||||
let application: Self = from_json(json)?;
|
||||
application
|
||||
.inputs
|
||||
.iter()
|
||||
.try_for_each(|input| validate_literal(&input.value))?;
|
||||
Ok(application)
|
||||
}
|
||||
|
||||
/// Encode the application with bytewise-sorted JSON keys.
|
||||
pub fn to_canonical_json(&self) -> Result<String> {
|
||||
self.inputs
|
||||
.iter()
|
||||
.try_for_each(|input| validate_literal(&input.value))?;
|
||||
canonical_json(self)
|
||||
}
|
||||
}
|
||||
|
||||
/// Stable table input bound to a registered parameter.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct InputBinding {
|
||||
pub parameter: String,
|
||||
pub field_id: i32,
|
||||
pub field_path: String,
|
||||
pub arrow_type: String,
|
||||
pub nullable: bool,
|
||||
}
|
||||
|
||||
/// Ordered result-field to table-field mapping for a grouped binding.
|
||||
///
|
||||
/// Assignment state is not part of the Slice 1 client contract. During the
|
||||
/// NULL transition there is no public Lance cell-flag identifier to persist.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct OutputMapping {
|
||||
pub result_field: String,
|
||||
pub output_name: String,
|
||||
pub output_field_id: i32,
|
||||
pub output_ordinal: u32,
|
||||
pub arrow_type: String,
|
||||
pub nullable: bool,
|
||||
}
|
||||
|
||||
/// Immutable grouped binding persisted by the Enterprise table service.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct FunctionBinding {
|
||||
binding_id: String,
|
||||
revision: u64,
|
||||
function: FunctionVersionRef,
|
||||
group_id: String,
|
||||
inputs: Vec<InputBinding>,
|
||||
outputs: Vec<OutputMapping>,
|
||||
}
|
||||
|
||||
impl FunctionBinding {
|
||||
pub fn binding_id(&self) -> &str {
|
||||
&self.binding_id
|
||||
}
|
||||
|
||||
pub fn revision(&self) -> u64 {
|
||||
self.revision
|
||||
}
|
||||
|
||||
pub fn function(&self) -> &FunctionVersionRef {
|
||||
&self.function
|
||||
}
|
||||
|
||||
pub fn group_id(&self) -> &str {
|
||||
&self.group_id
|
||||
}
|
||||
|
||||
pub fn inputs(&self) -> &[InputBinding] {
|
||||
&self.inputs
|
||||
}
|
||||
|
||||
pub fn outputs(&self) -> &[OutputMapping] {
|
||||
&self.outputs
|
||||
}
|
||||
}
|
||||
|
||||
impl_json!(FunctionBinding);
|
||||
|
||||
/// Stable terminal result of a remote Function-column refresh Job.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct RefreshColumnResult {
|
||||
pub rows_assigned: u64,
|
||||
pub rows_failed: u64,
|
||||
pub rows_remaining: u64,
|
||||
pub source_version: u64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub published_version: Option<u64>,
|
||||
}
|
||||
|
||||
impl RefreshColumnResult {
|
||||
/// Deprecated compatibility alias for `rows_assigned`.
|
||||
pub fn rows_filled(&self) -> u64 {
|
||||
self.rows_assigned
|
||||
}
|
||||
|
||||
/// Deprecated compatibility alias for `published_version`.
|
||||
pub fn version(&self) -> Option<u64> {
|
||||
self.published_version
|
||||
}
|
||||
}
|
||||
|
||||
impl_json!(RefreshColumnResult);
|
||||
+112
-22
@@ -6,6 +6,8 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde::de::DeserializeOwned;
|
||||
use serde_json::Value;
|
||||
use tokio::sync::watch;
|
||||
use tokio::task::{AbortHandle, JoinHandle};
|
||||
|
||||
@@ -19,43 +21,127 @@ pub(crate) trait JobHandle: Send + Sync {
|
||||
None
|
||||
}
|
||||
async fn status(&self) -> Result<String>;
|
||||
async fn wait(&self) -> Result<()>;
|
||||
async fn wait(&self) -> Result<TerminalResult>;
|
||||
async fn cancel(&self) -> Result<()>;
|
||||
}
|
||||
|
||||
/// A backend-neutral successful terminal result.
|
||||
///
|
||||
/// Local operations do not carry a value. Remote operations may carry JSON
|
||||
/// that the public [`Job`] decodes according to its result type.
|
||||
pub(crate) struct TerminalResult {
|
||||
#[allow(dead_code)] // Typed remote submit endpoints consume this after Slice 1.
|
||||
value: Option<Value>,
|
||||
#[allow(dead_code)] // Preserved so typed decode errors retain request correlation.
|
||||
request_id: Option<String>,
|
||||
}
|
||||
|
||||
impl TerminalResult {
|
||||
pub(crate) fn local() -> Self {
|
||||
Self {
|
||||
value: None,
|
||||
request_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn remote(value: Option<Value>, request_id: String) -> Self {
|
||||
Self {
|
||||
value,
|
||||
request_id: Some(request_id),
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)] // Exercised by the remote typed-result fixtures in Slice 1.
|
||||
fn decode<T: DeserializeOwned>(self) -> Result<T> {
|
||||
let request_id = self.request_id.unwrap_or_default();
|
||||
let value = self.value.ok_or_else(|| Error::Http {
|
||||
source: "successful typed job response did not contain a result".into(),
|
||||
request_id: request_id.clone(),
|
||||
status_code: None,
|
||||
})?;
|
||||
serde_json::from_value(value).map_err(|error| Error::Http {
|
||||
source: format!("failed to parse typed job result: {error}").into(),
|
||||
request_id,
|
||||
status_code: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type ResultDecoder<T> = fn(TerminalResult) -> Result<T>;
|
||||
|
||||
enum JobInner<T> {
|
||||
Handle {
|
||||
handle: Box<dyn JobHandle>,
|
||||
decode: ResultDecoder<T>,
|
||||
},
|
||||
Completed(T),
|
||||
}
|
||||
|
||||
/// A handle to an operation that may still be running.
|
||||
///
|
||||
/// The operation may already be complete when the handle is created.
|
||||
pub struct Job {
|
||||
handle: Option<Box<dyn JobHandle>>,
|
||||
pub struct Job<T = ()>
|
||||
where
|
||||
T: Clone + Send + Sync + 'static,
|
||||
{
|
||||
inner: JobInner<T>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for Job {
|
||||
impl<T> std::fmt::Debug for Job<T>
|
||||
where
|
||||
T: Clone + Send + Sync + 'static,
|
||||
{
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("Job")
|
||||
.field("id", &self.id())
|
||||
.field("done", &self.handle.is_none())
|
||||
.field("done", &matches!(self.inner, JobInner::Completed(_)))
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl Job {
|
||||
impl Job<()> {
|
||||
/// A job whose operation finished before the handle was created.
|
||||
pub(crate) fn new_done() -> Self {
|
||||
Self { handle: None }
|
||||
Self {
|
||||
inner: JobInner::Completed(()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn new(handle: Box<dyn JobHandle>) -> Self {
|
||||
Self {
|
||||
handle: Some(handle),
|
||||
inner: JobInner::Handle {
|
||||
handle,
|
||||
decode: |_| Ok(()),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// A job running as a task in this process.
|
||||
/// A unit-result job running as a task in this process.
|
||||
pub(crate) fn spawned(task: JoinHandle<Result<()>>) -> Self {
|
||||
Self::new(Box::new(SpawnedJob::new(task)))
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> Job<T>
|
||||
where
|
||||
T: Clone + DeserializeOwned + Send + Sync + 'static,
|
||||
{
|
||||
/// Construct a typed remote Job before result-specific submit APIs are added.
|
||||
#[allow(dead_code)]
|
||||
pub(crate) fn new_typed(handle: Box<dyn JobHandle>) -> Self {
|
||||
Self {
|
||||
inner: JobInner::Handle {
|
||||
handle,
|
||||
decode: TerminalResult::decode::<T>,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> Job<T>
|
||||
where
|
||||
T: Clone + Send + Sync + 'static,
|
||||
{
|
||||
/// Identifies the operation on the server that is running it.
|
||||
///
|
||||
/// Returned for correlating with server logs or the jobs API. Operations
|
||||
@@ -63,7 +149,10 @@ impl Job {
|
||||
/// value is opaque: parsing it or storing it to resume the job later is
|
||||
/// not supported.
|
||||
pub fn id(&self) -> Option<&str> {
|
||||
self.handle.as_ref().and_then(|handle| handle.id())
|
||||
match &self.inner {
|
||||
JobInner::Handle { handle, .. } => handle.id(),
|
||||
JobInner::Completed(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The operation's current lifecycle state: "running", "finished",
|
||||
@@ -73,9 +162,9 @@ impl Job {
|
||||
/// terminal failure state, or retry. States a newer server reports that
|
||||
/// this client version does not know pass through as-is.
|
||||
pub async fn status(&self) -> Result<String> {
|
||||
match &self.handle {
|
||||
None => Ok("finished".to_string()),
|
||||
Some(handle) => handle.status().await,
|
||||
match &self.inner {
|
||||
JobInner::Handle { handle, .. } => handle.status().await,
|
||||
JobInner::Completed(_) => Ok("finished".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -83,10 +172,10 @@ impl Job {
|
||||
///
|
||||
/// Returns [`crate::Error::JobFailed`] if the operation failed and
|
||||
/// [`crate::Error::JobCancelled`] if it was cancelled.
|
||||
pub async fn wait(&self) -> Result<()> {
|
||||
match &self.handle {
|
||||
None => Ok(()),
|
||||
Some(handle) => handle.wait().await,
|
||||
pub async fn wait(&self) -> Result<T> {
|
||||
match &self.inner {
|
||||
JobInner::Handle { handle, decode } => decode(handle.wait().await?),
|
||||
JobInner::Completed(result) => Ok(result.clone()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -94,9 +183,9 @@ impl Job {
|
||||
///
|
||||
/// Cancelling an operation that already finished is a no-op.
|
||||
pub async fn cancel(&self) -> Result<()> {
|
||||
match &self.handle {
|
||||
None => Ok(()),
|
||||
Some(handle) => handle.cancel().await,
|
||||
match &self.inner {
|
||||
JobInner::Handle { handle, .. } => handle.cancel().await,
|
||||
JobInner::Completed(_) => Ok(()),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -162,7 +251,7 @@ impl JobHandle for SpawnedJob {
|
||||
Ok(label.to_string())
|
||||
}
|
||||
|
||||
async fn wait(&self) -> Result<()> {
|
||||
async fn wait(&self) -> Result<TerminalResult> {
|
||||
let mut outcome = self.outcome.clone();
|
||||
let settled = outcome
|
||||
.wait_for(|outcome| outcome.is_some())
|
||||
@@ -172,7 +261,8 @@ impl JobHandle for SpawnedJob {
|
||||
})?
|
||||
.clone()
|
||||
.expect("wait_for returns once an outcome is set");
|
||||
settled.into_result()
|
||||
settled.into_result()?;
|
||||
Ok(TerminalResult::local())
|
||||
}
|
||||
|
||||
async fn cancel(&self) -> Result<()> {
|
||||
|
||||
@@ -181,6 +181,7 @@ pub mod dataloader;
|
||||
pub mod embeddings;
|
||||
pub mod error;
|
||||
pub mod expr;
|
||||
pub mod function;
|
||||
pub mod index;
|
||||
pub mod io;
|
||||
pub mod ipc;
|
||||
@@ -205,6 +206,7 @@ use serde::{Deserialize, Serialize};
|
||||
pub use blob::{BlobRangeRequest, blob, is_blob};
|
||||
pub use connection::{ConnectNamespaceBuilder, Connection};
|
||||
pub use error::{Error, JobFailure, Result};
|
||||
pub use function::FunctionVersion;
|
||||
pub use job::Job;
|
||||
use lance_index::vector::ApproxMode as LanceApproxMode;
|
||||
use lance_linalg::distance::DistanceType as LanceDistanceType;
|
||||
|
||||
@@ -25,7 +25,7 @@ use crate::database::{
|
||||
};
|
||||
use crate::error::Result;
|
||||
use crate::job::Job;
|
||||
use crate::remote::job::RemoteJob;
|
||||
use crate::remote::job::{DescribeJobResponse, RemoteJob, job_state_to_client};
|
||||
use crate::remote::util::stream_as_body;
|
||||
use crate::table::BaseTable;
|
||||
|
||||
@@ -472,48 +472,6 @@ struct RemoteListJobsResponse {
|
||||
page_token: Option<String>,
|
||||
}
|
||||
|
||||
/// The server's account of why a job failed. Absent from older servers,
|
||||
/// which report only the terminal state.
|
||||
#[derive(serde::Deserialize)]
|
||||
struct RemoteReportedFailure {
|
||||
#[serde(default)]
|
||||
phase: Option<String>,
|
||||
#[serde(default)]
|
||||
message: Option<String>,
|
||||
#[serde(default)]
|
||||
retryable: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct RemoteDescribeJobResponse {
|
||||
job_id: String,
|
||||
#[serde(default)]
|
||||
job_type: String,
|
||||
job_state: String,
|
||||
#[serde(default)]
|
||||
creation_ms: i64,
|
||||
#[serde(default)]
|
||||
spec: serde_json::Value,
|
||||
#[serde(default)]
|
||||
failure: Option<RemoteReportedFailure>,
|
||||
}
|
||||
|
||||
/// Server job states -> the client vocabulary ("running" / "finished" /
|
||||
/// "failed" / "cancelled"). Covers both the describe enum (IN_PROGRESS /
|
||||
/// DONE / FAILED / CANCELLED) and the registry's lowercase list-row states
|
||||
/// (in_progress / succeeded / failed / canceled / timed_out). States this
|
||||
/// client version does not know (e.g. created, queued) pass through as-is.
|
||||
fn job_state_to_client(state: &str) -> String {
|
||||
match state {
|
||||
"IN_PROGRESS" | "in_progress" => "running",
|
||||
"DONE" | "done" | "succeeded" => "finished",
|
||||
"FAILED" | "failed" | "TIMED_OUT" | "timed_out" => "failed",
|
||||
"CANCELLED" | "cancelled" | "canceled" => "cancelled",
|
||||
other => other,
|
||||
}
|
||||
.to_string()
|
||||
}
|
||||
|
||||
/// Bound on `list_jobs` page walking; a warning is logged when the listing
|
||||
/// is truncated at this many pages.
|
||||
const MAX_LIST_JOBS_PAGES: usize = 100;
|
||||
@@ -586,19 +544,14 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
|
||||
}) => return Ok(None),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
let body: RemoteDescribeJobResponse = rsp.json().await.err_to_http(request_id)?;
|
||||
let body: DescribeJobResponse = rsp.json().await.err_to_http(request_id)?;
|
||||
Ok(Some(JobDescription {
|
||||
job_id: body.job_id,
|
||||
job_type: body.job_type,
|
||||
state: job_state_to_client(&body.job_state),
|
||||
creation_ms: body.creation_ms,
|
||||
spec: body.spec,
|
||||
failure: body.failure.map(|reported| crate::error::JobFailure {
|
||||
phase: reported.phase,
|
||||
message: reported.message,
|
||||
retryable: reported.retryable,
|
||||
source: None,
|
||||
}),
|
||||
failure: body.failure.map(|reported| reported.into_job_failure()),
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -2507,7 +2460,7 @@ mod tests {
|
||||
http::Response::builder()
|
||||
.status(200)
|
||||
.body(format!(
|
||||
r#"{{"job_id": "job-1", "job_type": "create_index", "job_state": "{}", "creation_ms": 1}}"#,
|
||||
r#"{{"job_id": "job-1", "job_type": "create_function", "job_state": "{}", "creation_ms": 1, "result": {{"name": "embed", "version": "fv_1"}}}}"#,
|
||||
state
|
||||
))
|
||||
.unwrap()
|
||||
|
||||
+130
-30
@@ -8,10 +8,10 @@ use std::time::Duration;
|
||||
use async_trait::async_trait;
|
||||
use tokio::time::sleep;
|
||||
|
||||
use serde::{Deserialize, Deserializer};
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::error::{Error, JobFailure, Result};
|
||||
use crate::job::JobHandle;
|
||||
use crate::job::{JobHandle, TerminalResult};
|
||||
use crate::remote::client::{HttpSend, RequestResultExt, RestfulLanceDbClient};
|
||||
|
||||
/// Delay before the second job-state poll; doubles up to [`MAX_POLL_INTERVAL`].
|
||||
@@ -29,12 +29,6 @@ enum JobState {
|
||||
Other(String),
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for JobState {
|
||||
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> std::result::Result<Self, D::Error> {
|
||||
Ok(Self::from(String::deserialize(deserializer)?.as_str()))
|
||||
}
|
||||
}
|
||||
|
||||
impl JobState {
|
||||
/// The client vocabulary label for this state.
|
||||
fn client_label(&self) -> String {
|
||||
@@ -51,22 +45,26 @@ impl JobState {
|
||||
impl From<&str> for JobState {
|
||||
fn from(state: &str) -> Self {
|
||||
match state {
|
||||
"IN_PROGRESS" => Self::InProgress,
|
||||
"CANCELLED" => Self::Cancelled,
|
||||
"IN_PROGRESS" | "in_progress" => Self::InProgress,
|
||||
"CANCELLED" | "cancelled" | "canceled" => Self::Cancelled,
|
||||
// The server reports a timed-out job as FAILED on describe;
|
||||
// accept the raw registry state too in case a future server
|
||||
// stops folding it.
|
||||
"FAILED" | "TIMED_OUT" => Self::Failed,
|
||||
"DONE" => Self::Done,
|
||||
"FAILED" | "failed" | "TIMED_OUT" | "timed_out" => Self::Failed,
|
||||
"DONE" | "done" | "succeeded" => Self::Done,
|
||||
other => Self::Other(other.to_string()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn job_state_to_client(state: &str) -> String {
|
||||
JobState::from(state).client_label()
|
||||
}
|
||||
|
||||
/// The server's account of why a job failed. Absent from older servers, which
|
||||
/// report only the terminal state.
|
||||
#[derive(Deserialize)]
|
||||
struct ReportedFailure {
|
||||
pub(super) struct ReportedFailure {
|
||||
#[serde(default)]
|
||||
phase: Option<String>,
|
||||
#[serde(default)]
|
||||
@@ -75,11 +73,43 @@ struct ReportedFailure {
|
||||
retryable: Option<bool>,
|
||||
}
|
||||
|
||||
/// Forward-compatible `/v1/jobs/describe` wire envelope.
|
||||
#[derive(Deserialize)]
|
||||
struct DescribeJobResponse {
|
||||
job_state: JobState,
|
||||
pub(super) struct DescribeJobResponse {
|
||||
#[serde(default)]
|
||||
failure: Option<ReportedFailure>,
|
||||
pub(super) job_id: String,
|
||||
#[serde(default)]
|
||||
pub(super) job_type: String,
|
||||
pub(super) job_state: String,
|
||||
#[serde(default)]
|
||||
pub(super) creation_ms: i64,
|
||||
#[serde(default)]
|
||||
pub(super) spec: serde_json::Value,
|
||||
#[serde(default)]
|
||||
result: Option<serde_json::Value>,
|
||||
#[serde(default)]
|
||||
pub(super) failure: Option<ReportedFailure>,
|
||||
}
|
||||
|
||||
impl ReportedFailure {
|
||||
pub(super) fn into_job_failure(self) -> JobFailure {
|
||||
JobFailure {
|
||||
phase: self.phase,
|
||||
message: self.message,
|
||||
retryable: self.retryable,
|
||||
source: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl DescribeJobResponse {
|
||||
fn state(&self) -> JobState {
|
||||
JobState::from(self.job_state.as_str())
|
||||
}
|
||||
|
||||
fn into_terminal_result(self, request_id: String) -> TerminalResult {
|
||||
TerminalResult::remote(self.result, request_id)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct RemoteJob<S: HttpSend> {
|
||||
@@ -93,7 +123,7 @@ impl<S: HttpSend> RemoteJob<S> {
|
||||
}
|
||||
|
||||
/// One `/v1/jobs/describe` round trip.
|
||||
async fn describe(&self) -> Result<DescribeJobResponse> {
|
||||
async fn describe(&self) -> Result<(String, DescribeJobResponse)> {
|
||||
let request = self
|
||||
.client
|
||||
.post("/v1/jobs/describe")
|
||||
@@ -104,10 +134,10 @@ impl<S: HttpSend> RemoteJob<S> {
|
||||
let description: DescribeJobResponse =
|
||||
serde_json::from_str(&body).map_err(|e| Error::Http {
|
||||
source: format!("failed to parse job description: {}", e).into(),
|
||||
request_id,
|
||||
request_id: request_id.clone(),
|
||||
status_code: None,
|
||||
})?;
|
||||
Ok(description)
|
||||
Ok((request_id, description))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -118,26 +148,21 @@ impl<S: HttpSend> JobHandle for RemoteJob<S> {
|
||||
}
|
||||
|
||||
async fn status(&self) -> Result<String> {
|
||||
Ok(self.describe().await?.job_state.client_label())
|
||||
Ok(self.describe().await?.1.state().client_label())
|
||||
}
|
||||
|
||||
async fn wait(&self) -> Result<()> {
|
||||
async fn wait(&self) -> Result<TerminalResult> {
|
||||
let mut interval = INITIAL_POLL_INTERVAL;
|
||||
loop {
|
||||
let description = self.describe().await?;
|
||||
match description.job_state {
|
||||
JobState::Done => return Ok(()),
|
||||
let (request_id, description) = self.describe().await?;
|
||||
match description.state() {
|
||||
JobState::Done => return Ok(description.into_terminal_result(request_id)),
|
||||
JobState::Failed => {
|
||||
return Err(Error::JobFailed {
|
||||
job_id: Some(self.job_id.clone()),
|
||||
failure: description
|
||||
.failure
|
||||
.map(|reported| JobFailure {
|
||||
phase: reported.phase,
|
||||
message: reported.message,
|
||||
retryable: reported.retryable,
|
||||
source: None,
|
||||
})
|
||||
.map(ReportedFailure::into_job_failure)
|
||||
.unwrap_or_default(),
|
||||
});
|
||||
}
|
||||
@@ -168,3 +193,78 @@ impl<S: HttpSend> JobHandle for RemoteJob<S> {
|
||||
.map(|_| ())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use async_trait::async_trait;
|
||||
|
||||
use crate::Result;
|
||||
use crate::function::{FunctionVersion, RefreshColumnResult};
|
||||
use crate::job::{Job, JobHandle, TerminalResult};
|
||||
|
||||
use super::DescribeJobResponse;
|
||||
|
||||
const FUNCTION_JOB: &str =
|
||||
include_str!("../../tests/fixtures/first_class_functions/v1/remote_function_job.json");
|
||||
const REFRESH_JOB: &str =
|
||||
include_str!("../../tests/fixtures/first_class_functions/v1/remote_refresh_job.json");
|
||||
const UNIT_JOB: &str =
|
||||
include_str!("../../tests/fixtures/first_class_functions/v1/remote_unit_job.json");
|
||||
const MISSING_RESULT_JOB: &str = r#"{"job_state":"DONE"}"#;
|
||||
|
||||
struct FixtureRemoteJob(&'static str);
|
||||
|
||||
#[async_trait]
|
||||
impl JobHandle for FixtureRemoteJob {
|
||||
async fn status(&self) -> Result<String> {
|
||||
Ok("finished".to_string())
|
||||
}
|
||||
|
||||
async fn wait(&self) -> Result<TerminalResult> {
|
||||
let description: DescribeJobResponse =
|
||||
serde_json::from_str(self.0).expect("remote job fixture");
|
||||
Ok(description.into_terminal_result("fixture-request".to_string()))
|
||||
}
|
||||
|
||||
async fn cancel(&self) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn typed_remote_job_fixtures_decode_terminal_results() {
|
||||
let function = Job::<FunctionVersion>::new_typed(Box::new(FixtureRemoteJob(FUNCTION_JOB)));
|
||||
let result = function.wait().await.expect("typed FunctionVersion result");
|
||||
assert_eq!(result.version(), "fv_01K3EXACT");
|
||||
|
||||
let refresh =
|
||||
Job::<RefreshColumnResult>::new_typed(Box::new(FixtureRemoteJob(REFRESH_JOB)));
|
||||
let result = refresh.wait().await.expect("typed RefreshColumnResult");
|
||||
assert_eq!(result.rows_assigned, 999_998_800);
|
||||
assert_eq!(result.rows_filled(), result.rows_assigned);
|
||||
|
||||
let unit = Job::new(Box::new(FixtureRemoteJob(UNIT_JOB)));
|
||||
unit.wait()
|
||||
.await
|
||||
.expect("unit result ignores additive remote payloads");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn typed_remote_job_requires_a_terminal_result() {
|
||||
let typed =
|
||||
Job::<RefreshColumnResult>::new_typed(Box::new(FixtureRemoteJob(MISSING_RESULT_JOB)));
|
||||
let error = typed.wait().await.unwrap_err();
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("successful typed job response did not contain a result")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_wire_unknown_fields_are_forward_decodable() {
|
||||
let response: DescribeJobResponse =
|
||||
serde_json::from_str(FUNCTION_JOB).expect("function job fixture");
|
||||
assert_eq!(response.job_state, "DONE");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -162,13 +162,13 @@ impl<S: HttpSend> crate::job::JobHandle for FreshnessJob<S> {
|
||||
crate::job::JobHandle::status(&self.inner).await
|
||||
}
|
||||
|
||||
async fn wait(&self) -> Result<()> {
|
||||
crate::job::JobHandle::wait(&self.inner).await?;
|
||||
async fn wait(&self) -> Result<crate::job::TerminalResult> {
|
||||
let result = crate::job::JobHandle::wait(&self.inner).await?;
|
||||
let version = self.version.read().await;
|
||||
if version.is_none() {
|
||||
self.freshness.lock().unwrap().checkout_baseline = Some(SystemTime::now());
|
||||
}
|
||||
Ok(())
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn cancel(&self) -> Result<()> {
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use lancedb::function::{
|
||||
FunctionApplication, FunctionBinding, FunctionVersion, RefreshColumnResult,
|
||||
};
|
||||
use serde_json::Value;
|
||||
|
||||
fn fixture(name: &str) -> String {
|
||||
let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests/fixtures/first_class_functions/v1")
|
||||
.join(name);
|
||||
fs::read_to_string(path).expect("fixture must be readable")
|
||||
}
|
||||
|
||||
fn job_result(name: &str) -> Value {
|
||||
serde_json::from_str::<Value>(&fixture(name)).expect("remote Job fixture")["result"].clone()
|
||||
}
|
||||
|
||||
fn assert_no_secret_values(value: &Value) {
|
||||
match value {
|
||||
Value::Object(values) => {
|
||||
for (key, value) in values {
|
||||
assert!(
|
||||
!matches!(
|
||||
key.as_str(),
|
||||
"secret_value" | "secret_values" | "resolved_secret" | "resolved_secrets"
|
||||
),
|
||||
"client canonical value must not model resolved secret material"
|
||||
);
|
||||
assert_no_secret_values(value);
|
||||
}
|
||||
}
|
||||
Value::Array(values) => values.iter().for_each(assert_no_secret_values),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn function_version_job_result_matches_shared_canonical_golden() {
|
||||
let result = job_result("remote_function_job.json");
|
||||
let version = FunctionVersion::from_json(&result.to_string()).expect("FunctionVersion result");
|
||||
|
||||
assert_eq!(version.name(), "embed");
|
||||
assert_eq!(version.version(), "fv_01K3EXACT");
|
||||
assert_eq!(version.runtime_digest(), "sha256:runtime");
|
||||
assert_eq!(version.required_secrets(), &["HF_TOKEN"]);
|
||||
assert_eq!(
|
||||
version.to_canonical_json().expect("canonical JSON"),
|
||||
fixture("remote_function_version.canonical.json").trim()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn version_identity_is_immutable_and_exact() {
|
||||
let original = job_result("remote_function_job.json");
|
||||
let version =
|
||||
FunctionVersion::from_json(&original.to_string()).expect("FunctionVersion result");
|
||||
let reopened = version.clone();
|
||||
assert_eq!(reopened, version);
|
||||
assert_eq!(reopened.name(), version.name());
|
||||
assert_eq!(reopened.version(), version.version());
|
||||
|
||||
let mut changed = original;
|
||||
changed["version"] = Value::String("fv_01K3DIFFERENT".to_string());
|
||||
let changed = FunctionVersion::from_json(&changed.to_string()).expect("changed version");
|
||||
assert_ne!(changed, version);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn application_and_binding_match_shared_remote_goldens() {
|
||||
let application = FunctionApplication::from_json(&fixture("remote_function_application.json"))
|
||||
.expect("application fixture");
|
||||
assert_eq!(application.function().version, "fv_01K3TEXT");
|
||||
assert_eq!(application.output().kind, "named_struct");
|
||||
assert_eq!(application.inputs().len(), 2);
|
||||
assert_eq!(
|
||||
application.to_canonical_json().expect("canonical JSON"),
|
||||
fixture("remote_function_application.canonical.json").trim()
|
||||
);
|
||||
|
||||
let binding = FunctionBinding::from_json(&fixture("remote_function_binding.json"))
|
||||
.expect("binding fixture");
|
||||
assert_eq!(binding.revision(), 3);
|
||||
assert_eq!(binding.function().version, "fv_01K3TEXT");
|
||||
assert_eq!(binding.outputs()[0].output_ordinal, 0);
|
||||
assert_eq!(binding.outputs()[1].output_ordinal, 1);
|
||||
assert_eq!(
|
||||
binding.to_canonical_json().expect("canonical JSON"),
|
||||
fixture("remote_function_binding.canonical.json").trim()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refresh_job_result_matches_shared_canonical_golden() {
|
||||
let result = job_result("remote_refresh_job.json");
|
||||
let result = RefreshColumnResult::from_json(&result.to_string()).expect("refresh result");
|
||||
assert_eq!(result.rows_assigned, 999_998_800);
|
||||
assert_eq!(result.rows_filled(), result.rows_assigned);
|
||||
assert_eq!(result.version(), result.published_version);
|
||||
assert_eq!(
|
||||
result.to_canonical_json().expect("canonical JSON"),
|
||||
fixture("remote_refresh_result.canonical.json").trim()
|
||||
);
|
||||
|
||||
let result = RefreshColumnResult::from_json(&fixture(
|
||||
"remote_refresh_result_without_published_version.json",
|
||||
))
|
||||
.expect("optional version");
|
||||
assert_eq!(result.published_version, None);
|
||||
assert_eq!(
|
||||
result
|
||||
.to_canonical_json()
|
||||
.expect("canonical result without version"),
|
||||
fixture("remote_refresh_result_without_published_version.canonical.json").trim()
|
||||
);
|
||||
assert_eq!(
|
||||
RefreshColumnResult::from_json(
|
||||
&result
|
||||
.to_canonical_json()
|
||||
.expect("canonical result without version")
|
||||
)
|
||||
.expect("round-trip result without version"),
|
||||
result
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_fields_and_discriminators_are_forward_decodable() {
|
||||
let mut result = job_result("remote_function_job.json");
|
||||
result["future_version_metadata"] = serde_json::json!({"retention_class": "catalog"});
|
||||
result["runtime"] = serde_json::json!({
|
||||
"kind": "wasm",
|
||||
"module_digest": "sha256:wasm"
|
||||
});
|
||||
result["signature"]["output"]["kind"] = Value::String("future_output_shape".to_string());
|
||||
|
||||
let version = FunctionVersion::from_json(&result.to_string()).expect("future remote value");
|
||||
assert_eq!(version.runtime().kind(), "wasm");
|
||||
assert_eq!(version.runtime().python_version(), None);
|
||||
assert_eq!(version.signature().output.kind, "future_output_shape");
|
||||
assert_eq!(
|
||||
serde_json::from_str::<Value>(
|
||||
&version.to_canonical_json().expect("canonical future value")
|
||||
)
|
||||
.expect("canonical JSON")["runtime"],
|
||||
serde_json::json!({"kind": "wasm"})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn floating_point_application_literals_are_rejected_consistently() {
|
||||
let error = FunctionApplication::from_json(&fixture("remote_function_application_float.json"))
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("floating-point Function literals")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_client_values_contain_secret_names_only() {
|
||||
let result = job_result("remote_function_job.json");
|
||||
let version = FunctionVersion::from_json(&result.to_string()).expect("FunctionVersion result");
|
||||
let canonical: Value = serde_json::from_str(
|
||||
&version
|
||||
.to_canonical_json()
|
||||
.expect("canonical FunctionVersion"),
|
||||
)
|
||||
.expect("canonical JSON");
|
||||
|
||||
assert_eq!(
|
||||
canonical["required_secrets"],
|
||||
serde_json::json!(["HF_TOKEN"])
|
||||
);
|
||||
assert_no_secret_values(&canonical);
|
||||
}
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
{"columns":{"normalized_text":"search_text","token_count":"search_token_count"},"function":{"name":"text_features","version":"fv_01K3TEXT"},"group_id":"fg_01K3TEXT","inputs":[{"kind":"column","parameter":"title","value":{"path":"title"}},{"kind":"column","parameter":"body","value":{"path":"body"}}],"output":{"fields":[{"arrow_type":"utf8","name":"normalized_text","nullable":false},{"arrow_type":"int64","name":"token_count","nullable":false}],"kind":"named_struct"}}
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"function": {"name": "text_features", "version": "fv_01K3TEXT"},
|
||||
"inputs": [
|
||||
{"parameter": "title", "kind": "column", "value": {"path": "title"}},
|
||||
{"parameter": "body", "kind": "column", "value": {"path": "body"}}
|
||||
],
|
||||
"output": {
|
||||
"kind": "named_struct",
|
||||
"fields": [
|
||||
{"name": "normalized_text", "arrow_type": "utf8", "nullable": false},
|
||||
{"name": "token_count", "arrow_type": "int64", "nullable": false}
|
||||
]
|
||||
},
|
||||
"group_id": "fg_01K3TEXT",
|
||||
"columns": {
|
||||
"normalized_text": "search_text",
|
||||
"token_count": "search_token_count"
|
||||
},
|
||||
"future_application": {"declaration_mode": "managed"}
|
||||
}
|
||||
Vendored
+8
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"function": {"name": "score", "version": "fv_01K3FLOAT"},
|
||||
"inputs": [
|
||||
{"parameter": "threshold", "kind": "literal", "value": 1e-7}
|
||||
],
|
||||
"output": {"kind": "scalar", "arrow_type": "bool", "nullable": false},
|
||||
"group_id": "fg_01K3FLOAT"
|
||||
}
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
{"binding_id":"fb_01K3TEXT","function":{"name":"text_features","version":"fv_01K3TEXT"},"group_id":"fg_01K3TEXT","inputs":[{"arrow_type":"utf8","field_id":11,"field_path":"title","nullable":true,"parameter":"title"},{"arrow_type":"utf8","field_id":12,"field_path":"body","nullable":true,"parameter":"body"}],"outputs":[{"arrow_type":"utf8","nullable":false,"output_field_id":21,"output_name":"search_text","output_ordinal":0,"result_field":"normalized_text"},{"arrow_type":"int64","nullable":false,"output_field_id":22,"output_name":"search_token_count","output_ordinal":1,"result_field":"token_count"}],"revision":3}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"binding_id": "fb_01K3TEXT",
|
||||
"revision": 3,
|
||||
"function": {"name": "text_features", "version": "fv_01K3TEXT"},
|
||||
"group_id": "fg_01K3TEXT",
|
||||
"inputs": [
|
||||
{"parameter": "title", "field_id": 11, "field_path": "title", "arrow_type": "utf8", "nullable": true},
|
||||
{"parameter": "body", "field_id": 12, "field_path": "body", "arrow_type": "utf8", "nullable": true}
|
||||
],
|
||||
"outputs": [
|
||||
{"result_field": "normalized_text", "output_name": "search_text", "output_field_id": 21, "output_ordinal": 0, "arrow_type": "utf8", "nullable": false},
|
||||
{"result_field": "token_count", "output_name": "search_token_count", "output_field_id": 22, "output_ordinal": 1, "arrow_type": "int64", "nullable": false}
|
||||
],
|
||||
"future_binding": {"metadata_revision": 1}
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"job_id": "job_function_01K3",
|
||||
"job_type": "create_function",
|
||||
"job_state": "DONE",
|
||||
"creation_ms": 1787270400000,
|
||||
"spec": {"name": "embed"},
|
||||
"result": {
|
||||
"name": "embed",
|
||||
"version": "fv_01K3EXACT",
|
||||
"artifact": {
|
||||
"kind": "python_callable",
|
||||
"digest": "sha256:code",
|
||||
"entrypoint": "embed"
|
||||
},
|
||||
"signature": {
|
||||
"inputs": [{"name": "text", "arrow_type": "utf8", "nullable": true}],
|
||||
"output": {"kind": "scalar", "arrow_type": "list<float32>", "nullable": false}
|
||||
},
|
||||
"runtime": {
|
||||
"kind": "python",
|
||||
"python_version": "3.12",
|
||||
"environment": {"kind": "pip", "packages": ["sentence-transformers>=3"]},
|
||||
"env": {"TOKENIZERS_PARALLELISM": "false"}
|
||||
},
|
||||
"runtime_digest": "sha256:runtime",
|
||||
"environment_digest": "sha256:environment",
|
||||
"required_secrets": ["HF_TOKEN"],
|
||||
"created_at": "2026-08-21T00:00:00Z"
|
||||
},
|
||||
"future_job": {"trace_id": "trace-1"}
|
||||
}
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
{"artifact":{"digest":"sha256:code","entrypoint":"embed","kind":"python_callable"},"created_at":"2026-08-21T00:00:00Z","environment_digest":"sha256:environment","name":"embed","required_secrets":["HF_TOKEN"],"runtime":{"env":{"TOKENIZERS_PARALLELISM":"false"},"environment":{"kind":"pip","packages":["sentence-transformers>=3"]},"kind":"python","python_version":"3.12"},"runtime_digest":"sha256:runtime","signature":{"inputs":[{"arrow_type":"utf8","name":"text","nullable":true}],"output":{"arrow_type":"list<float32>","kind":"scalar","nullable":false}},"version":"fv_01K3EXACT"}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"job_id": "job_refresh_01K3",
|
||||
"job_type": "refresh_function_columns",
|
||||
"job_state": "DONE",
|
||||
"creation_ms": 1787270400001,
|
||||
"spec": {"table": "documents", "binding_revision": 3},
|
||||
"result": {
|
||||
"rows_assigned": 999998800,
|
||||
"rows_failed": 0,
|
||||
"rows_remaining": 0,
|
||||
"source_version": 812,
|
||||
"published_version": 919,
|
||||
"future_result": {"committed_fragment_groups": 100}
|
||||
}
|
||||
}
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"published_version":919,"rows_assigned":999998800,"rows_failed":0,"rows_remaining":0,"source_version":812}
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"rows_assigned":120,"rows_failed":0,"rows_remaining":0,"source_version":812}
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"rows_assigned": 120,
|
||||
"rows_failed": 0,
|
||||
"rows_remaining": 0,
|
||||
"source_version": 812
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"job_id": "job_index_01K3",
|
||||
"job_type": "create_index",
|
||||
"job_state": "DONE",
|
||||
"creation_ms": 1787270400002,
|
||||
"spec": {"column": "vector"},
|
||||
"result": {"future_information": "ignored by Job<()>"},
|
||||
"future_job": {"trace_id": "trace-2"}
|
||||
}
|
||||
Reference in New Issue
Block a user