mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-16 07:36:50 +00:00
feat(remote): expose canonical job statistics reports (#5016)
Expose `Job::stats()` in Rust and synchronous/asynchronous Python jobs. It reads `POST /v1/jobs/stats` and returns the canonical server-rendered report verbatim, or `None` for a running job or one without a readable durable profile. It does not wait for completion or recompute measurements. On `cc25005525671756b5cf5ebd96492b792e9aa1cb`, all three Rust transport tests and six Python tests passed against the matching rebuilt native extension; strict SDK Clippy also passed. These tests cover optional reports, exact text preservation, fresh reads, errors, and the HTTP boundary with a controlled server. Against the actual Phalanx HTTP route from [Sophon #10247](https://github.com/lancedb/sophon/pull/10247) (`5be3236`), the same native extension also completed two sync and two async reads after durable registry archival, preserving the exact canonical report. This is an HTTP/registry integration fixture, not a Function workload or shared-cluster acceptance. The private runner's JSON field-order collector failed after the successful probe; all raw evidence was retained and hash-verified, and the original receipt was recovered with the corrected parser without rerunning the probe. The follow-up `ad36f66` only corrects the synchronous Python API-reference link to the asynchronous method. Ruff checks and formatting passed; runtime evidence above remains applicable. The actual Phalanx/native-SDK probe covers archived reports; running-job null availability is covered by separate backend and SDK transport tests, not that end-to-end probe. Part of ENT-3501.
This commit is contained in:
@@ -437,6 +437,7 @@ class Job:
|
||||
async def cancel(self) -> None: ...
|
||||
async def refresh(self) -> None: ...
|
||||
async def timeline(self) -> str: ...
|
||||
async def stats(self) -> Optional[str]: ...
|
||||
async def events(
|
||||
self,
|
||||
*,
|
||||
|
||||
@@ -425,6 +425,32 @@ class AsyncJob(Generic[T]):
|
||||
description = self._inner._description if self._inner is not None else None
|
||||
return getattr(description, name) if description is not None else None
|
||||
|
||||
async def stats(self) -> Optional[str]:
|
||||
"""Return a printable run report for this Function job.
|
||||
|
||||
The report includes recorded phase timings, finishing-supervisor slot
|
||||
occupancy, all-attempt split execution and batch latency, and accepted-row
|
||||
throughput. Each call reads the current server report.
|
||||
Returns ``None`` while the job is running or if no recorded profile is
|
||||
available, without waiting for completion. Missing measurements remain
|
||||
unavailable rather than becoming zero. The server formats the report;
|
||||
the SDK returns it verbatim.
|
||||
|
||||
Requires a server-side Function job and a server supporting run reports.
|
||||
Local operations and SQL query handles do not support this operation.
|
||||
Unsupported operations raise ``NotImplementedError`` or a server error.
|
||||
This method returns text; use ``print`` to display it.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> print(await job.stats()) # doctest: +SKIP
|
||||
"""
|
||||
if self._inner is None:
|
||||
raise NotImplementedError(
|
||||
"job statistics require a server-side Function job"
|
||||
)
|
||||
return await self._inner.stats()
|
||||
|
||||
async def export_timeline(self, path: str | PathLike[str]) -> None:
|
||||
"""Write a fresh recorded Function job timeline as Chrome trace JSON.
|
||||
|
||||
@@ -649,6 +675,21 @@ class Job(Generic[T]):
|
||||
def _field(self, name: str) -> Optional[Any]:
|
||||
return getattr(self._inner, name) if self._inner is not None else None
|
||||
|
||||
def stats(self) -> Optional[str]:
|
||||
"""Return a printable Function job run report.
|
||||
|
||||
See [AsyncJob.stats][lancedb.job.AsyncJob.stats] for metrics and availability.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> print(job.stats()) # doctest: +SKIP
|
||||
"""
|
||||
if self._inner is None:
|
||||
raise NotImplementedError(
|
||||
"job statistics require a server-side Function job"
|
||||
)
|
||||
return LOOP.run(self._inner.stats())
|
||||
|
||||
def export_timeline(self, path: str | PathLike[str]) -> None:
|
||||
"""Write Chrome trace JSON for this Function job.
|
||||
|
||||
|
||||
@@ -754,6 +754,46 @@ async def test_query_wait_timeout_requires_keeping_original_handle():
|
||||
assert "db.open_job" not in message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("sync", [False, True])
|
||||
async def test_stats_returns_fresh_report_without_waiting_or_printing(sync, capsys):
|
||||
reports = [
|
||||
None,
|
||||
"Function job: finished\nWorker utilization: unavailable\n",
|
||||
"Function job: finished\nWorker utilization: 25.0%\n",
|
||||
]
|
||||
inner = SimpleNamespace(
|
||||
stats=AsyncMock(side_effect=reports),
|
||||
wait=AsyncMock(),
|
||||
cancel=AsyncMock(),
|
||||
)
|
||||
job = AsyncJob(inner)
|
||||
for expected in reports:
|
||||
actual = await asyncio.to_thread(Job(job).stats) if sync else await job.stats()
|
||||
assert actual == expected
|
||||
inner.wait.assert_not_awaited()
|
||||
inner.cancel.assert_not_awaited()
|
||||
assert capsys.readouterr().out == ""
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("sync", [False, True])
|
||||
async def test_stats_preserves_errors_and_rejects_untracked_jobs(sync):
|
||||
error = RuntimeError("run reports are unavailable on this server")
|
||||
job = AsyncJob(SimpleNamespace(stats=AsyncMock(side_effect=error)))
|
||||
with pytest.raises(RuntimeError) as exc:
|
||||
if sync:
|
||||
await asyncio.to_thread(Job(job).stats)
|
||||
else:
|
||||
await job.stats()
|
||||
assert exc.value is error
|
||||
with pytest.raises(NotImplementedError, match="server-side Function job"):
|
||||
if sync:
|
||||
Job(None).stats()
|
||||
else:
|
||||
await AsyncJob(None).stats()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("sync", [False, True])
|
||||
async def test_export_timeline_writes_fresh_recording_without_waiting(sync, tmp_path):
|
||||
|
||||
@@ -2732,6 +2732,50 @@ async def test_remote_job_wait_disconnection_timeout_can_be_resumed(sync):
|
||||
assert exc.value.__cause__.is_transport_error is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("sync", [False, True])
|
||||
async def test_remote_job_stats_returns_native_optional_report(sync):
|
||||
report = "Function job: finished\nWorker utilization: unavailable\n"
|
||||
paths = []
|
||||
|
||||
def handler(request):
|
||||
payload = json.loads(request.rfile.read(int(request.headers["Content-Length"])))
|
||||
assert payload == {"job_id": "refresh-1"}
|
||||
paths.append(request.path)
|
||||
if request.path == "/v1/jobs/describe":
|
||||
response = {
|
||||
"job_id": "refresh-1",
|
||||
"job_type": "function_refresh",
|
||||
"job_state": "IN_PROGRESS",
|
||||
"creation_ms": 1000,
|
||||
}
|
||||
elif request.path == "/v1/jobs/stats":
|
||||
available = paths.count("/v1/jobs/stats") > 1
|
||||
response = {
|
||||
"job_id": "refresh-1",
|
||||
"profile": {"future_field": True} if available else None,
|
||||
"report": report if available else None,
|
||||
}
|
||||
else:
|
||||
raise AssertionError(f"unexpected request: {request.path}")
|
||||
request.send_response(200)
|
||||
request.send_header("Content-Type", "application/json")
|
||||
request.end_headers()
|
||||
request.wfile.write(json.dumps(response).encode())
|
||||
|
||||
if sync:
|
||||
with mock_lancedb_connection(handler) as db:
|
||||
job = db.open_job("refresh-1")
|
||||
assert job.stats() is None
|
||||
assert job.stats() == report
|
||||
else:
|
||||
async with mock_lancedb_connection_async(handler) as db:
|
||||
job = await db.open_job("refresh-1")
|
||||
assert await job.stats() is None
|
||||
assert await job.stats() == report
|
||||
assert paths == ["/v1/jobs/describe", "/v1/jobs/stats", "/v1/jobs/stats"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("sync", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -159,6 +159,11 @@ impl Job {
|
||||
})
|
||||
}
|
||||
|
||||
pub fn stats(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.inner.clone();
|
||||
future_into_py(self_.py(), async move { inner.stats().await.infer_error() })
|
||||
}
|
||||
|
||||
#[pyo3(signature = (*, limit=None, filter=None))]
|
||||
pub fn events(
|
||||
self_: PyRef<'_, Self>,
|
||||
|
||||
Reference in New Issue
Block a user