fix(python): show Function registration progress and image details (#5040)

Report registration lifecycle and client elapsed time through existing job handles, and provide concise image inspection. Preserve quiet wait behavior, environment-lock forwarding, failure diagnostics and timeout recovery.
This commit is contained in:
Jack Ye
2026-11-15 02:05:06 -08:00
committed by GitHub
parent 1feec84d2b
commit 3aed07c396
6 changed files with 396 additions and 13 deletions
+19
View File
@@ -424,6 +424,25 @@ code that used the former LanceDB Enterprise-only argument order.
## Functions and Jobs
Use `progress=True` when waiting for Function registration in a notebook or
terminal. A callback receives structured snapshots instead:
```python
fn = db.create_function(my_function, progress=True)
registration = db.create_function_async(other_function)
fn = registration.wait(progress=lambda snapshot: print(snapshot))
print(fn.image.describe())
```
Registration uses the existing SQL query handle. The current handle reports
lifecycle state and `client_elapsed_time_millis` since the handle was created;
detailed build phases and build logs are unavailable. Keep the same handle after
a wait timeout to resume waiting. Failures retain the server error and request
identifiers. `fn.image` has a concise representation; `describe()` exposes Python
and CUDA requirements without inspecting Pydantic internals. Platforms, base image
and size are `None` until the service reports them. The full image descriptor and
canonical JSON remain available.
A `@function` definition is also a local callable. Call a decorated function
directly with row inputs, or call a decorated class with its initialization fields
and then call the returned instance; call `close()` if the class defines it. This
+9 -2
View File
@@ -1051,6 +1051,7 @@ class DBConnection(EnforceOverrides):
exist_ok: bool = False,
namespace_path: Optional[List[str]] = None,
refresh_lock: bool = False,
progress: Any = False,
) -> FunctionVersion:
"""Build and register a scalar Python function, then return its version.
@@ -1076,6 +1077,9 @@ class DBConnection(EnforceOverrides):
refresh_lock : bool, default False
Ask the server to resolve a fresh environment lock instead of
reusing the published version's lock.
progress : bool, callable, or tqdm-like object, default False
Display lifecycle state and elapsed time, or receive structured
snapshots through a callback. See [Job.wait][lancedb.job.Job.wait].
When the registration job resolves, each resolved-environment warning
reported by the server is emitted as a Python ``UserWarning``. Catalog
@@ -1095,13 +1099,16 @@ class DBConnection(EnforceOverrides):
)
```
"""
return self.create_function_async(
job = self.create_function_async(
definition,
secrets=secrets,
exist_ok=exist_ok,
namespace_path=namespace_path,
refresh_lock=refresh_lock,
).wait()
)
if progress is False or progress is None:
return job.wait()
return job.wait(progress=progress)
def create_function_async(
self,
+58 -1
View File
@@ -544,13 +544,70 @@ class PythonRuntimeSpec(_RemoteValue):
class FunctionImage(_RemoteValue):
"""A complete OCI Function image identified by its exact manifest digest."""
"""A complete OCI Function image identified by its exact manifest digest.
``describe()`` and the concise representation inspect the descriptor without
walking Pydantic model internals. Missing metadata is ``None`` (unknown).
The current service does not report image platforms, base image or byte size;
these cannot be inferred from the client's platform or the manifest digest.
The full descriptor remains available through ``descriptor``.
"""
manifest_digest: str
descriptor: Mapping[str, Any]
source: bool
models: tuple[HuggingFaceModel, ...] = ()
def describe(self) -> dict[str, Any]:
"""Return concise image details without fetching registry content.
``requires_gpu`` reflects a ``lance.cuda`` requirement in a v1
descriptor, rather than promising compatibility with every GPU.
``platforms``, ``base_image`` and ``size_bytes`` are unavailable from
the current wire contract and return ``None``.
Examples
--------
>>> print(fn.image.describe()) # doctest: +SKIP
"""
python = self.descriptor.get("python")
requires = self.descriptor.get("requires")
capabilities = (
requires.get("capabilities") if isinstance(requires, Mapping) else None
)
known_capabilities = (
self.descriptor.get("format_version") == 1
and isinstance(capabilities, (list, tuple))
and all(isinstance(item, Mapping) for item in capabilities)
)
cuda = (
next(
(item for item in capabilities if item.get("type") == "lance.cuda"),
None,
)
if known_capabilities
else None
)
python_version = python.get("version") if isinstance(python, Mapping) else None
cuda_toolkit = cuda.get("cuda_toolkit") if cuda is not None else None
return {
"manifest_digest": self.manifest_digest,
"python_version": python_version
if isinstance(python_version, str)
else None,
"requires_gpu": cuda is not None if known_capabilities else None,
"cuda_toolkit": cuda_toolkit if isinstance(cuda_toolkit, str) else None,
"platforms": None,
"base_image": None,
"size_bytes": None,
}
def __repr_args__(self):
for name, value in self.describe().items():
if isinstance(value, str) and len(value) > 80:
value = value[:77] + "..."
yield name, value
class ResolvedPackage(_RemoteValue):
"""One installed Python distribution measured from a baked environment.
+68 -7
View File
@@ -4,8 +4,10 @@
"""Handles to asynchronous registry jobs, SQL queries, and local operations."""
import asyncio
import logging
import reprlib
import time
from contextlib import suppress
from datetime import timedelta
from os import PathLike
from pathlib import Path
@@ -53,11 +55,13 @@ def _timeout_message(
)
def _normalize_job_progress(progress):
def _normalize_job_progress(progress, *, query=False):
"""Normalize ``wait(progress=...)`` using the same convention as writes."""
if progress is True:
from tqdm.auto import tqdm
if query:
return tqdm(bar_format="Query{postfix} [{elapsed}]"), True
return tqdm(unit=" rows"), True
if progress is False or progress is None:
return None, False
@@ -65,6 +69,7 @@ def _normalize_job_progress(progress):
_ETA_MAX_SAMPLE_INTERVAL_MILLIS = 10 * 60 * 1_000
_QUERY_PROGRESS_INTERVAL_SECONDS = 1.0
def _progress_eta(previous, current):
@@ -203,6 +208,7 @@ class AsyncJob(Generic[T]):
self._inner = inner
self._result_decoder = result_decoder
self._last_progress_snapshot = None
self._created_at = time.monotonic()
def _enrich_progress(self, snapshot):
enriched = _with_progress_eta(self._last_progress_snapshot, snapshot)
@@ -235,17 +241,29 @@ class AsyncJob(Generic[T]):
async def progress(self) -> Optional[dict[str, Any]]:
"""Fetch a structured progress snapshot.
Registry jobs always report lifecycle fields. Queue metadata and
counters are included when supplied by the server. Older servers may
Registry jobs always report lifecycle fields. SQL query handles report
lifecycle state and ``client_elapsed_time_millis``, measured since this
handle was created. They do not expose server build phases or logs.
Queue metadata and counters are included when supplied by the server.
Older servers may
omit counters, and in-process operations return ``None``. A missing
counter is unknown rather than zero. This is additive to :meth:`status`,
whose string contract is unchanged.
"""
if self._inner is None:
return None
await self._inner.status()
state = await self._inner.status()
description = getattr(self._inner, "_description", None)
if description is None or description.progress is None:
if self.job_type == "query":
return {
"job_id": self.id,
"job_type": "query",
"state": state,
"client_elapsed_time_millis": int(
(time.monotonic() - self._created_at) * 1_000
),
}
return None
return self._enrich_progress(description.progress)
@@ -270,6 +288,13 @@ class AsyncJob(Generic[T]):
new execution epoch or smaller plan resets the bar to that plan.
``False`` preserves the original wait behavior.
For SQL-backed Function registration, progress shows the query's
lifecycle state and client elapsed time. It observes the existing
query handle while its original waiter runs; it does not resubmit or
cancel the registration. Detailed bake phases and build logs are not
available from the current query handle. Failure exceptions retain the
server's cause and request identifiers.
Raises `JobFailedError` if the operation failed, `JobCancelledError`
if it was cancelled, and `TimeoutError` if `timeout` elapses first.
Transient connection and read errors are retried with capped exponential
@@ -285,7 +310,10 @@ class AsyncJob(Generic[T]):
"""
if self._inner is None:
return cast(T, None)
progress_obj, owns = _normalize_job_progress(progress)
query_progress = (
progress is not False and progress is not None and self.job_type == "query"
)
progress_obj, owns = _normalize_job_progress(progress, query=query_progress)
progress_callback = _job_progress_callback(progress_obj, self._enrich_progress)
last_error = None
@@ -311,13 +339,46 @@ class AsyncJob(Generic[T]):
await asyncio.sleep(delay)
delay = min(delay * 2, 30.0)
async def observe_query():
async def report():
try:
snapshot = await self.progress()
if snapshot is not None:
progress_callback(snapshot)
except Exception:
logging.getLogger(__name__).warning(
"job progress observer failed", exc_info=True
)
waiter = asyncio.create_task(wait_with_retries())
try:
while not waiter.done():
await report()
await asyncio.wait(
{waiter}, timeout=_QUERY_PROGRESS_INTERVAL_SECONDS
)
try:
return await waiter
finally:
await report()
finally:
if not waiter.done():
waiter.cancel()
with suppress(asyncio.CancelledError):
await waiter
wait_for_result = (
observe_query
if query_progress and progress_callback is not None
else wait_with_retries
)
try:
try:
if timeout is None:
result = await wait_with_retries()
result = await wait_for_result()
else:
result = await asyncio.wait_for(
wait_with_retries(), timeout.total_seconds()
wait_for_result(), timeout.total_seconds()
)
except asyncio.TimeoutError as error:
if last_error is None:
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import json
import warnings
from collections import UserDict
from pathlib import Path
from types import MappingProxyType
@@ -13,6 +14,7 @@ import lancedb.functions as functions
from lancedb.functions import (
FunctionApplication,
FunctionBinding,
FunctionImage,
FunctionVersion,
PythonRuntimeSpec,
SecretBinding,
@@ -149,6 +151,59 @@ def test_function_version_identity_is_immutable_and_exact():
FunctionVersion(**{**value, "version": invalid})
@pytest.mark.parametrize("gpu", [False, True])
def test_image_inspection_is_concise_warning_free_and_preserves_wire_state(gpu):
value = job_result("remote_function_job.json")["image"]
value["descriptor"]["future_metadata"] = "x" * 100_000
if gpu:
value["descriptor"]["requires"]["capabilities"] = [
{
"type": "lance.cuda",
"version": 1,
"device_count": 1,
"cuda_toolkit": "12.4",
}
]
image = FunctionImage(**value)
with warnings.catch_warnings():
warnings.simplefilter("error", DeprecationWarning)
details = image.describe()
rendered = repr(image)
assert "manifest_digest=" in str(image)
encoded = json.loads(image.to_canonical_json())
assert details == {
"manifest_digest": value["manifest_digest"],
"python_version": "3.12.14",
"requires_gpu": gpu,
"cuda_toolkit": "12.4" if gpu else None,
"platforms": None,
"base_image": None,
"size_bytes": None,
}
assert len(rendered) < 350
assert f"requires_gpu={gpu}" in rendered
assert "future_metadata" not in rendered
assert encoded == value
@pytest.mark.parametrize(
"descriptor",
[
{},
{"format_version": 2, "requires": {"capabilities": []}},
{"format_version": 1, "requires": {"capabilities": [None]}},
{"python": {"version": {"future": "x" * 100_000}}},
],
)
def test_image_inspection_leaves_missing_or_unrecognized_metadata_unknown(descriptor):
image = FunctionImage(
manifest_digest="sha256:example", descriptor=descriptor, source=True
)
assert image.describe()["requires_gpu"] is None
assert image.describe()["python_version"] is None
assert len(repr(image)) < 350
@pytest.mark.parametrize(
"case", json.loads(fixture("runtime_identity.json")), ids=lambda case: case["name"]
)
+187 -3
View File
@@ -11,6 +11,7 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
from lancedb.exceptions import JobCancelledError, JobFailedError
from lancedb.db import DBConnection
from lancedb.job import AsyncJob, Job, _progress_eta
from lancedb.remote.errors import HttpError, RetryError
@@ -729,15 +730,198 @@ async def test_wait_progress_bar_tracks_late_planning_without_regressing(
@pytest.mark.asyncio
async def test_wait_progress_false_keeps_legacy_inner_wait_call():
@pytest.mark.parametrize("progress", [False, None])
async def test_wait_progress_false_keeps_legacy_inner_wait_call(progress):
class NoProgressArgument:
id = "job-legacy"
job_type = None
@property
def job_type(self):
raise AssertionError("quiet waits must not inspect progress metadata")
async def wait(self):
return None
assert await AsyncJob(NoProgressArgument()).wait(progress=False) is None
assert await AsyncJob(NoProgressArgument()).wait(progress=progress) is None
class _RegistrationQuery:
id = "registration-query"
job_type = "query"
_description = None
def __init__(self, error=None):
self.ready = asyncio.Event()
self.error = error
self.state = "running"
self.wait_calls = 0
self.cancel_calls = 0
async def status(self):
return self.state
async def wait(self, progress=None):
self.wait_calls += 1
await self.ready.wait()
if self.error is not None:
self.state = (
"cancelled" if isinstance(self.error, JobCancelledError) else "failed"
)
raise self.error
self.state = "finished"
return '{"version": "1"}'
async def cancel(self):
self.cancel_calls += 1
@pytest.mark.asyncio
@pytest.mark.parametrize("sync", [False, True])
async def test_query_progress_observes_one_registration_waiter(sync, monkeypatch):
monkeypatch.setattr("lancedb.job._QUERY_PROGRESS_INTERVAL_SECONDS", 0)
inner = _RegistrationQuery()
job = AsyncJob(inner, json.loads)
updates = []
def observe(snapshot):
updates.append(snapshot)
if len(updates) == 2:
inner.ready.set()
if sync:
result = await asyncio.to_thread(Job(job).wait, progress=observe)
else:
result = await job.wait(progress=observe)
assert result == {"version": "1"}
assert inner.wait_calls == 1
assert inner.cancel_calls == 0
assert updates[0]["state"] == "running"
assert updates[-1]["state"] == "finished"
assert all(snapshot["job_id"] == inner.id for snapshot in updates)
elapsed = [snapshot["client_elapsed_time_millis"] for snapshot in updates]
assert elapsed == sorted(elapsed)
assert elapsed[0] >= 0
assert all("phase" not in snapshot for snapshot in updates)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"error",
[
JobFailedError("bake failed", job_id="registration-query"),
JobCancelledError("cancelled"),
],
)
async def test_query_progress_preserves_terminal_exceptions(error):
inner = _RegistrationQuery(error)
inner.ready.set()
updates = []
with pytest.raises(type(error)) as raised:
await AsyncJob(inner).wait(progress=updates.append)
assert raised.value is error
assert updates[-1]["state"] == inner.state
assert inner.wait_calls == 1
@pytest.mark.asyncio
async def test_query_progress_display_has_no_row_counter_and_closes_on_failure(
monkeypatch,
):
tqdm_module = pytest.importorskip("tqdm.auto")
bar = SimpleNamespace(
n=0,
total=None,
update=MagicMock(),
set_postfix_str=MagicMock(),
close=MagicMock(),
)
constructor = MagicMock(return_value=bar)
monkeypatch.setattr(tqdm_module, "tqdm", constructor)
inner = _RegistrationQuery(JobFailedError("bake failed"))
inner.ready.set()
with pytest.raises(JobFailedError, match="bake failed"):
await AsyncJob(inner).wait(progress=True)
assert "elapsed" in constructor.call_args.kwargs["bar_format"]
assert "rows" not in str(constructor.call_args)
assert bar.set_postfix_str.call_args.args == ("failed",)
bar.close.assert_called_once_with()
@pytest.mark.asyncio
@pytest.mark.parametrize("failed", [False, True])
@pytest.mark.parametrize("observer_error", ["status", "callback"])
async def test_query_progress_observer_errors_do_not_replace_result_or_server_failure(
failed, observer_error, caplog
):
error = (
JobFailedError("bake failed", job_id="registration-query") if failed else None
)
inner = _RegistrationQuery(error)
inner.ready.set()
if observer_error == "status":
inner.status = AsyncMock(side_effect=RuntimeError("observer failed"))
def observe(snapshot):
if observer_error == "callback":
raise RuntimeError("observer failed")
job = AsyncJob(inner, json.loads)
if failed:
with pytest.raises(JobFailedError) as raised:
await job.wait(progress=observe)
assert raised.value is error
else:
assert await job.wait(progress=observe) == {"version": "1"}
assert inner.wait_calls == 1
assert inner.cancel_calls == 0
assert "job progress observer failed" in caplog.text
@pytest.mark.asyncio
async def test_query_progress_timeout_keeps_registration_available_on_same_handle():
inner = _RegistrationQuery()
job = AsyncJob(inner, json.loads)
updates = []
with pytest.raises(TimeoutError, match="keep this handle"):
await job.wait(timeout=timedelta(seconds=0.01), progress=updates.append)
assert inner.state == "running"
assert inner.cancel_calls == 0
inner.ready.set()
assert await job.wait(progress=updates.append) == {"version": "1"}
assert inner.wait_calls == 2
assert updates[-1]["state"] == "finished"
@pytest.mark.parametrize("refresh_lock", [False, True])
def test_blocking_create_function_forwards_progress_to_registration_handle(
refresh_lock,
):
connection = MagicMock()
definition = object()
progress = object()
result = DBConnection.create_function(
connection, definition, refresh_lock=refresh_lock, progress=progress
)
connection.create_function_async.assert_called_once_with(
definition,
secrets=None,
exist_ok=False,
namespace_path=None,
refresh_lock=refresh_lock,
)
connection.create_function_async.return_value.wait.assert_called_once_with(
progress=progress
)
assert result is connection.create_function_async.return_value.wait.return_value
@pytest.mark.parametrize("progress", [False, None])
def test_blocking_create_function_preserves_quiet_wait_call(progress):
connection = MagicMock()
result = DBConnection.create_function(connection, object(), progress=progress)
connection.create_function_async.return_value.wait.assert_called_once_with()
assert result is connection.create_function_async.return_value.wait.return_value
@pytest.mark.asyncio