mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-16 07:36:50 +00:00
fix(python): show Function registration progress and image details (#5040)
Report registration lifecycle and client elapsed time through existing job handles, and provide concise image inspection. Preserve quiet wait behavior, environment-lock forwarding, failure diagnostics and timeout recovery.
This commit is contained in:
@@ -1051,6 +1051,7 @@ class DBConnection(EnforceOverrides):
|
||||
exist_ok: bool = False,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
refresh_lock: bool = False,
|
||||
progress: Any = False,
|
||||
) -> FunctionVersion:
|
||||
"""Build and register a scalar Python function, then return its version.
|
||||
|
||||
@@ -1076,6 +1077,9 @@ class DBConnection(EnforceOverrides):
|
||||
refresh_lock : bool, default False
|
||||
Ask the server to resolve a fresh environment lock instead of
|
||||
reusing the published version's lock.
|
||||
progress : bool, callable, or tqdm-like object, default False
|
||||
Display lifecycle state and elapsed time, or receive structured
|
||||
snapshots through a callback. See [Job.wait][lancedb.job.Job.wait].
|
||||
|
||||
When the registration job resolves, each resolved-environment warning
|
||||
reported by the server is emitted as a Python ``UserWarning``. Catalog
|
||||
@@ -1095,13 +1099,16 @@ class DBConnection(EnforceOverrides):
|
||||
)
|
||||
```
|
||||
"""
|
||||
return self.create_function_async(
|
||||
job = self.create_function_async(
|
||||
definition,
|
||||
secrets=secrets,
|
||||
exist_ok=exist_ok,
|
||||
namespace_path=namespace_path,
|
||||
refresh_lock=refresh_lock,
|
||||
).wait()
|
||||
)
|
||||
if progress is False or progress is None:
|
||||
return job.wait()
|
||||
return job.wait(progress=progress)
|
||||
|
||||
def create_function_async(
|
||||
self,
|
||||
|
||||
@@ -544,13 +544,70 @@ class PythonRuntimeSpec(_RemoteValue):
|
||||
|
||||
|
||||
class FunctionImage(_RemoteValue):
|
||||
"""A complete OCI Function image identified by its exact manifest digest."""
|
||||
"""A complete OCI Function image identified by its exact manifest digest.
|
||||
|
||||
``describe()`` and the concise representation inspect the descriptor without
|
||||
walking Pydantic model internals. Missing metadata is ``None`` (unknown).
|
||||
The current service does not report image platforms, base image or byte size;
|
||||
these cannot be inferred from the client's platform or the manifest digest.
|
||||
The full descriptor remains available through ``descriptor``.
|
||||
"""
|
||||
|
||||
manifest_digest: str
|
||||
descriptor: Mapping[str, Any]
|
||||
source: bool
|
||||
models: tuple[HuggingFaceModel, ...] = ()
|
||||
|
||||
def describe(self) -> dict[str, Any]:
|
||||
"""Return concise image details without fetching registry content.
|
||||
|
||||
``requires_gpu`` reflects a ``lance.cuda`` requirement in a v1
|
||||
descriptor, rather than promising compatibility with every GPU.
|
||||
``platforms``, ``base_image`` and ``size_bytes`` are unavailable from
|
||||
the current wire contract and return ``None``.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> print(fn.image.describe()) # doctest: +SKIP
|
||||
"""
|
||||
python = self.descriptor.get("python")
|
||||
requires = self.descriptor.get("requires")
|
||||
capabilities = (
|
||||
requires.get("capabilities") if isinstance(requires, Mapping) else None
|
||||
)
|
||||
known_capabilities = (
|
||||
self.descriptor.get("format_version") == 1
|
||||
and isinstance(capabilities, (list, tuple))
|
||||
and all(isinstance(item, Mapping) for item in capabilities)
|
||||
)
|
||||
cuda = (
|
||||
next(
|
||||
(item for item in capabilities if item.get("type") == "lance.cuda"),
|
||||
None,
|
||||
)
|
||||
if known_capabilities
|
||||
else None
|
||||
)
|
||||
python_version = python.get("version") if isinstance(python, Mapping) else None
|
||||
cuda_toolkit = cuda.get("cuda_toolkit") if cuda is not None else None
|
||||
return {
|
||||
"manifest_digest": self.manifest_digest,
|
||||
"python_version": python_version
|
||||
if isinstance(python_version, str)
|
||||
else None,
|
||||
"requires_gpu": cuda is not None if known_capabilities else None,
|
||||
"cuda_toolkit": cuda_toolkit if isinstance(cuda_toolkit, str) else None,
|
||||
"platforms": None,
|
||||
"base_image": None,
|
||||
"size_bytes": None,
|
||||
}
|
||||
|
||||
def __repr_args__(self):
|
||||
for name, value in self.describe().items():
|
||||
if isinstance(value, str) and len(value) > 80:
|
||||
value = value[:77] + "..."
|
||||
yield name, value
|
||||
|
||||
|
||||
class ResolvedPackage(_RemoteValue):
|
||||
"""One installed Python distribution measured from a baked environment.
|
||||
|
||||
@@ -4,8 +4,10 @@
|
||||
"""Handles to asynchronous registry jobs, SQL queries, and local operations."""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import reprlib
|
||||
import time
|
||||
from contextlib import suppress
|
||||
from datetime import timedelta
|
||||
from os import PathLike
|
||||
from pathlib import Path
|
||||
@@ -53,11 +55,13 @@ def _timeout_message(
|
||||
)
|
||||
|
||||
|
||||
def _normalize_job_progress(progress):
|
||||
def _normalize_job_progress(progress, *, query=False):
|
||||
"""Normalize ``wait(progress=...)`` using the same convention as writes."""
|
||||
if progress is True:
|
||||
from tqdm.auto import tqdm
|
||||
|
||||
if query:
|
||||
return tqdm(bar_format="Query{postfix} [{elapsed}]"), True
|
||||
return tqdm(unit=" rows"), True
|
||||
if progress is False or progress is None:
|
||||
return None, False
|
||||
@@ -65,6 +69,7 @@ def _normalize_job_progress(progress):
|
||||
|
||||
|
||||
_ETA_MAX_SAMPLE_INTERVAL_MILLIS = 10 * 60 * 1_000
|
||||
_QUERY_PROGRESS_INTERVAL_SECONDS = 1.0
|
||||
|
||||
|
||||
def _progress_eta(previous, current):
|
||||
@@ -203,6 +208,7 @@ class AsyncJob(Generic[T]):
|
||||
self._inner = inner
|
||||
self._result_decoder = result_decoder
|
||||
self._last_progress_snapshot = None
|
||||
self._created_at = time.monotonic()
|
||||
|
||||
def _enrich_progress(self, snapshot):
|
||||
enriched = _with_progress_eta(self._last_progress_snapshot, snapshot)
|
||||
@@ -235,17 +241,29 @@ class AsyncJob(Generic[T]):
|
||||
async def progress(self) -> Optional[dict[str, Any]]:
|
||||
"""Fetch a structured progress snapshot.
|
||||
|
||||
Registry jobs always report lifecycle fields. Queue metadata and
|
||||
counters are included when supplied by the server. Older servers may
|
||||
Registry jobs always report lifecycle fields. SQL query handles report
|
||||
lifecycle state and ``client_elapsed_time_millis``, measured since this
|
||||
handle was created. They do not expose server build phases or logs.
|
||||
Queue metadata and counters are included when supplied by the server.
|
||||
Older servers may
|
||||
omit counters, and in-process operations return ``None``. A missing
|
||||
counter is unknown rather than zero. This is additive to :meth:`status`,
|
||||
whose string contract is unchanged.
|
||||
"""
|
||||
if self._inner is None:
|
||||
return None
|
||||
await self._inner.status()
|
||||
state = await self._inner.status()
|
||||
description = getattr(self._inner, "_description", None)
|
||||
if description is None or description.progress is None:
|
||||
if self.job_type == "query":
|
||||
return {
|
||||
"job_id": self.id,
|
||||
"job_type": "query",
|
||||
"state": state,
|
||||
"client_elapsed_time_millis": int(
|
||||
(time.monotonic() - self._created_at) * 1_000
|
||||
),
|
||||
}
|
||||
return None
|
||||
return self._enrich_progress(description.progress)
|
||||
|
||||
@@ -270,6 +288,13 @@ class AsyncJob(Generic[T]):
|
||||
new execution epoch or smaller plan resets the bar to that plan.
|
||||
``False`` preserves the original wait behavior.
|
||||
|
||||
For SQL-backed Function registration, progress shows the query's
|
||||
lifecycle state and client elapsed time. It observes the existing
|
||||
query handle while its original waiter runs; it does not resubmit or
|
||||
cancel the registration. Detailed bake phases and build logs are not
|
||||
available from the current query handle. Failure exceptions retain the
|
||||
server's cause and request identifiers.
|
||||
|
||||
Raises `JobFailedError` if the operation failed, `JobCancelledError`
|
||||
if it was cancelled, and `TimeoutError` if `timeout` elapses first.
|
||||
Transient connection and read errors are retried with capped exponential
|
||||
@@ -285,7 +310,10 @@ class AsyncJob(Generic[T]):
|
||||
"""
|
||||
if self._inner is None:
|
||||
return cast(T, None)
|
||||
progress_obj, owns = _normalize_job_progress(progress)
|
||||
query_progress = (
|
||||
progress is not False and progress is not None and self.job_type == "query"
|
||||
)
|
||||
progress_obj, owns = _normalize_job_progress(progress, query=query_progress)
|
||||
progress_callback = _job_progress_callback(progress_obj, self._enrich_progress)
|
||||
last_error = None
|
||||
|
||||
@@ -311,13 +339,46 @@ class AsyncJob(Generic[T]):
|
||||
await asyncio.sleep(delay)
|
||||
delay = min(delay * 2, 30.0)
|
||||
|
||||
async def observe_query():
|
||||
async def report():
|
||||
try:
|
||||
snapshot = await self.progress()
|
||||
if snapshot is not None:
|
||||
progress_callback(snapshot)
|
||||
except Exception:
|
||||
logging.getLogger(__name__).warning(
|
||||
"job progress observer failed", exc_info=True
|
||||
)
|
||||
|
||||
waiter = asyncio.create_task(wait_with_retries())
|
||||
try:
|
||||
while not waiter.done():
|
||||
await report()
|
||||
await asyncio.wait(
|
||||
{waiter}, timeout=_QUERY_PROGRESS_INTERVAL_SECONDS
|
||||
)
|
||||
try:
|
||||
return await waiter
|
||||
finally:
|
||||
await report()
|
||||
finally:
|
||||
if not waiter.done():
|
||||
waiter.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await waiter
|
||||
|
||||
wait_for_result = (
|
||||
observe_query
|
||||
if query_progress and progress_callback is not None
|
||||
else wait_with_retries
|
||||
)
|
||||
try:
|
||||
try:
|
||||
if timeout is None:
|
||||
result = await wait_with_retries()
|
||||
result = await wait_for_result()
|
||||
else:
|
||||
result = await asyncio.wait_for(
|
||||
wait_with_retries(), timeout.total_seconds()
|
||||
wait_for_result(), timeout.total_seconds()
|
||||
)
|
||||
except asyncio.TimeoutError as error:
|
||||
if last_error is None:
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import json
|
||||
import warnings
|
||||
from collections import UserDict
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
@@ -13,6 +14,7 @@ import lancedb.functions as functions
|
||||
from lancedb.functions import (
|
||||
FunctionApplication,
|
||||
FunctionBinding,
|
||||
FunctionImage,
|
||||
FunctionVersion,
|
||||
PythonRuntimeSpec,
|
||||
SecretBinding,
|
||||
@@ -149,6 +151,59 @@ def test_function_version_identity_is_immutable_and_exact():
|
||||
FunctionVersion(**{**value, "version": invalid})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("gpu", [False, True])
|
||||
def test_image_inspection_is_concise_warning_free_and_preserves_wire_state(gpu):
|
||||
value = job_result("remote_function_job.json")["image"]
|
||||
value["descriptor"]["future_metadata"] = "x" * 100_000
|
||||
if gpu:
|
||||
value["descriptor"]["requires"]["capabilities"] = [
|
||||
{
|
||||
"type": "lance.cuda",
|
||||
"version": 1,
|
||||
"device_count": 1,
|
||||
"cuda_toolkit": "12.4",
|
||||
}
|
||||
]
|
||||
image = FunctionImage(**value)
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("error", DeprecationWarning)
|
||||
details = image.describe()
|
||||
rendered = repr(image)
|
||||
assert "manifest_digest=" in str(image)
|
||||
encoded = json.loads(image.to_canonical_json())
|
||||
assert details == {
|
||||
"manifest_digest": value["manifest_digest"],
|
||||
"python_version": "3.12.14",
|
||||
"requires_gpu": gpu,
|
||||
"cuda_toolkit": "12.4" if gpu else None,
|
||||
"platforms": None,
|
||||
"base_image": None,
|
||||
"size_bytes": None,
|
||||
}
|
||||
assert len(rendered) < 350
|
||||
assert f"requires_gpu={gpu}" in rendered
|
||||
assert "future_metadata" not in rendered
|
||||
assert encoded == value
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"descriptor",
|
||||
[
|
||||
{},
|
||||
{"format_version": 2, "requires": {"capabilities": []}},
|
||||
{"format_version": 1, "requires": {"capabilities": [None]}},
|
||||
{"python": {"version": {"future": "x" * 100_000}}},
|
||||
],
|
||||
)
|
||||
def test_image_inspection_leaves_missing_or_unrecognized_metadata_unknown(descriptor):
|
||||
image = FunctionImage(
|
||||
manifest_digest="sha256:example", descriptor=descriptor, source=True
|
||||
)
|
||||
assert image.describe()["requires_gpu"] is None
|
||||
assert image.describe()["python_version"] is None
|
||||
assert len(repr(image)) < 350
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case", json.loads(fixture("runtime_identity.json")), ids=lambda case: case["name"]
|
||||
)
|
||||
|
||||
@@ -11,6 +11,7 @@ from unittest.mock import AsyncMock, MagicMock
|
||||
import pytest
|
||||
|
||||
from lancedb.exceptions import JobCancelledError, JobFailedError
|
||||
from lancedb.db import DBConnection
|
||||
from lancedb.job import AsyncJob, Job, _progress_eta
|
||||
from lancedb.remote.errors import HttpError, RetryError
|
||||
|
||||
@@ -729,15 +730,198 @@ async def test_wait_progress_bar_tracks_late_planning_without_regressing(
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wait_progress_false_keeps_legacy_inner_wait_call():
|
||||
@pytest.mark.parametrize("progress", [False, None])
|
||||
async def test_wait_progress_false_keeps_legacy_inner_wait_call(progress):
|
||||
class NoProgressArgument:
|
||||
id = "job-legacy"
|
||||
job_type = None
|
||||
|
||||
@property
|
||||
def job_type(self):
|
||||
raise AssertionError("quiet waits must not inspect progress metadata")
|
||||
|
||||
async def wait(self):
|
||||
return None
|
||||
|
||||
assert await AsyncJob(NoProgressArgument()).wait(progress=False) is None
|
||||
assert await AsyncJob(NoProgressArgument()).wait(progress=progress) is None
|
||||
|
||||
|
||||
class _RegistrationQuery:
|
||||
id = "registration-query"
|
||||
job_type = "query"
|
||||
_description = None
|
||||
|
||||
def __init__(self, error=None):
|
||||
self.ready = asyncio.Event()
|
||||
self.error = error
|
||||
self.state = "running"
|
||||
self.wait_calls = 0
|
||||
self.cancel_calls = 0
|
||||
|
||||
async def status(self):
|
||||
return self.state
|
||||
|
||||
async def wait(self, progress=None):
|
||||
self.wait_calls += 1
|
||||
await self.ready.wait()
|
||||
if self.error is not None:
|
||||
self.state = (
|
||||
"cancelled" if isinstance(self.error, JobCancelledError) else "failed"
|
||||
)
|
||||
raise self.error
|
||||
self.state = "finished"
|
||||
return '{"version": "1"}'
|
||||
|
||||
async def cancel(self):
|
||||
self.cancel_calls += 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("sync", [False, True])
|
||||
async def test_query_progress_observes_one_registration_waiter(sync, monkeypatch):
|
||||
monkeypatch.setattr("lancedb.job._QUERY_PROGRESS_INTERVAL_SECONDS", 0)
|
||||
inner = _RegistrationQuery()
|
||||
job = AsyncJob(inner, json.loads)
|
||||
updates = []
|
||||
|
||||
def observe(snapshot):
|
||||
updates.append(snapshot)
|
||||
if len(updates) == 2:
|
||||
inner.ready.set()
|
||||
|
||||
if sync:
|
||||
result = await asyncio.to_thread(Job(job).wait, progress=observe)
|
||||
else:
|
||||
result = await job.wait(progress=observe)
|
||||
assert result == {"version": "1"}
|
||||
assert inner.wait_calls == 1
|
||||
assert inner.cancel_calls == 0
|
||||
assert updates[0]["state"] == "running"
|
||||
assert updates[-1]["state"] == "finished"
|
||||
assert all(snapshot["job_id"] == inner.id for snapshot in updates)
|
||||
elapsed = [snapshot["client_elapsed_time_millis"] for snapshot in updates]
|
||||
assert elapsed == sorted(elapsed)
|
||||
assert elapsed[0] >= 0
|
||||
assert all("phase" not in snapshot for snapshot in updates)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"error",
|
||||
[
|
||||
JobFailedError("bake failed", job_id="registration-query"),
|
||||
JobCancelledError("cancelled"),
|
||||
],
|
||||
)
|
||||
async def test_query_progress_preserves_terminal_exceptions(error):
|
||||
inner = _RegistrationQuery(error)
|
||||
inner.ready.set()
|
||||
updates = []
|
||||
with pytest.raises(type(error)) as raised:
|
||||
await AsyncJob(inner).wait(progress=updates.append)
|
||||
assert raised.value is error
|
||||
assert updates[-1]["state"] == inner.state
|
||||
assert inner.wait_calls == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_progress_display_has_no_row_counter_and_closes_on_failure(
|
||||
monkeypatch,
|
||||
):
|
||||
tqdm_module = pytest.importorskip("tqdm.auto")
|
||||
bar = SimpleNamespace(
|
||||
n=0,
|
||||
total=None,
|
||||
update=MagicMock(),
|
||||
set_postfix_str=MagicMock(),
|
||||
close=MagicMock(),
|
||||
)
|
||||
constructor = MagicMock(return_value=bar)
|
||||
monkeypatch.setattr(tqdm_module, "tqdm", constructor)
|
||||
inner = _RegistrationQuery(JobFailedError("bake failed"))
|
||||
inner.ready.set()
|
||||
|
||||
with pytest.raises(JobFailedError, match="bake failed"):
|
||||
await AsyncJob(inner).wait(progress=True)
|
||||
assert "elapsed" in constructor.call_args.kwargs["bar_format"]
|
||||
assert "rows" not in str(constructor.call_args)
|
||||
assert bar.set_postfix_str.call_args.args == ("failed",)
|
||||
bar.close.assert_called_once_with()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("failed", [False, True])
|
||||
@pytest.mark.parametrize("observer_error", ["status", "callback"])
|
||||
async def test_query_progress_observer_errors_do_not_replace_result_or_server_failure(
|
||||
failed, observer_error, caplog
|
||||
):
|
||||
error = (
|
||||
JobFailedError("bake failed", job_id="registration-query") if failed else None
|
||||
)
|
||||
inner = _RegistrationQuery(error)
|
||||
inner.ready.set()
|
||||
if observer_error == "status":
|
||||
inner.status = AsyncMock(side_effect=RuntimeError("observer failed"))
|
||||
|
||||
def observe(snapshot):
|
||||
if observer_error == "callback":
|
||||
raise RuntimeError("observer failed")
|
||||
|
||||
job = AsyncJob(inner, json.loads)
|
||||
if failed:
|
||||
with pytest.raises(JobFailedError) as raised:
|
||||
await job.wait(progress=observe)
|
||||
assert raised.value is error
|
||||
else:
|
||||
assert await job.wait(progress=observe) == {"version": "1"}
|
||||
assert inner.wait_calls == 1
|
||||
assert inner.cancel_calls == 0
|
||||
assert "job progress observer failed" in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_progress_timeout_keeps_registration_available_on_same_handle():
|
||||
inner = _RegistrationQuery()
|
||||
job = AsyncJob(inner, json.loads)
|
||||
updates = []
|
||||
with pytest.raises(TimeoutError, match="keep this handle"):
|
||||
await job.wait(timeout=timedelta(seconds=0.01), progress=updates.append)
|
||||
assert inner.state == "running"
|
||||
assert inner.cancel_calls == 0
|
||||
inner.ready.set()
|
||||
assert await job.wait(progress=updates.append) == {"version": "1"}
|
||||
assert inner.wait_calls == 2
|
||||
assert updates[-1]["state"] == "finished"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("refresh_lock", [False, True])
|
||||
def test_blocking_create_function_forwards_progress_to_registration_handle(
|
||||
refresh_lock,
|
||||
):
|
||||
connection = MagicMock()
|
||||
definition = object()
|
||||
progress = object()
|
||||
result = DBConnection.create_function(
|
||||
connection, definition, refresh_lock=refresh_lock, progress=progress
|
||||
)
|
||||
connection.create_function_async.assert_called_once_with(
|
||||
definition,
|
||||
secrets=None,
|
||||
exist_ok=False,
|
||||
namespace_path=None,
|
||||
refresh_lock=refresh_lock,
|
||||
)
|
||||
connection.create_function_async.return_value.wait.assert_called_once_with(
|
||||
progress=progress
|
||||
)
|
||||
assert result is connection.create_function_async.return_value.wait.return_value
|
||||
|
||||
|
||||
@pytest.mark.parametrize("progress", [False, None])
|
||||
def test_blocking_create_function_preserves_quiet_wait_call(progress):
|
||||
connection = MagicMock()
|
||||
result = DBConnection.create_function(connection, object(), progress=progress)
|
||||
connection.create_function_async.return_value.wait.assert_called_once_with()
|
||||
assert result is connection.create_function_async.return_value.wait.return_value
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
Reference in New Issue
Block a user