feat(remote): expose canonical job statistics reports (#5016)

Expose `Job::stats()` in Rust and synchronous/asynchronous Python jobs.
It reads `POST /v1/jobs/stats` and returns the canonical server-rendered
report verbatim, or `None` for a running job or one without a readable
durable profile. It does not wait for completion or recompute
measurements.

On `cc25005525671756b5cf5ebd96492b792e9aa1cb`, all three Rust transport
tests and six Python tests passed against the matching rebuilt native
extension; strict SDK Clippy also passed. These tests cover optional
reports, exact text preservation, fresh reads, errors, and the HTTP
boundary with a controlled server. Against the actual Phalanx HTTP route
from [Sophon #10247](https://github.com/lancedb/sophon/pull/10247)
(`5be3236`), the same native extension also completed two sync and two
async reads after durable registry archival, preserving the exact
canonical report. This is an HTTP/registry integration fixture, not a
Function workload or shared-cluster acceptance. The private runner's
JSON field-order collector failed after the successful probe; all raw
evidence was retained and hash-verified, and the original receipt was
recovered with the corrected parser without rerunning the probe.

The follow-up `ad36f66` only corrects the synchronous Python
API-reference link to the asynchronous method. Ruff checks and
formatting passed; runtime evidence above remains applicable. The actual
Phalanx/native-SDK probe covers archived reports; running-job null
availability is covered by separate backend and SDK transport tests, not
that end-to-end probe.

Part of ENT-3501.
This commit is contained in:
Xuanwo
2026-11-13 23:41:33 -08:00
committed by GitHub
parent d67ec47921
commit 97e62dc11b
7 changed files with 256 additions and 0 deletions
+1
View File
@@ -437,6 +437,7 @@ class Job:
async def cancel(self) -> None: ...
async def refresh(self) -> None: ...
async def timeline(self) -> str: ...
async def stats(self) -> Optional[str]: ...
async def events(
self,
*,
+41
View File
@@ -425,6 +425,32 @@ class AsyncJob(Generic[T]):
description = self._inner._description if self._inner is not None else None
return getattr(description, name) if description is not None else None
async def stats(self) -> Optional[str]:
"""Return a printable run report for this Function job.
The report includes recorded phase timings, finishing-supervisor slot
occupancy, all-attempt split execution and batch latency, and accepted-row
throughput. Each call reads the current server report.
Returns ``None`` while the job is running or if no recorded profile is
available, without waiting for completion. Missing measurements remain
unavailable rather than becoming zero. The server formats the report;
the SDK returns it verbatim.
Requires a server-side Function job and a server supporting run reports.
Local operations and SQL query handles do not support this operation.
Unsupported operations raise ``NotImplementedError`` or a server error.
This method returns text; use ``print`` to display it.
Examples
--------
>>> print(await job.stats()) # doctest: +SKIP
"""
if self._inner is None:
raise NotImplementedError(
"job statistics require a server-side Function job"
)
return await self._inner.stats()
async def export_timeline(self, path: str | PathLike[str]) -> None:
"""Write a fresh recorded Function job timeline as Chrome trace JSON.
@@ -649,6 +675,21 @@ class Job(Generic[T]):
def _field(self, name: str) -> Optional[Any]:
return getattr(self._inner, name) if self._inner is not None else None
def stats(self) -> Optional[str]:
"""Return a printable Function job run report.
See [AsyncJob.stats][lancedb.job.AsyncJob.stats] for metrics and availability.
Examples
--------
>>> print(job.stats()) # doctest: +SKIP
"""
if self._inner is None:
raise NotImplementedError(
"job statistics require a server-side Function job"
)
return LOOP.run(self._inner.stats())
def export_timeline(self, path: str | PathLike[str]) -> None:
"""Write Chrome trace JSON for this Function job.
+40
View File
@@ -754,6 +754,46 @@ async def test_query_wait_timeout_requires_keeping_original_handle():
assert "db.open_job" not in message
@pytest.mark.asyncio
@pytest.mark.parametrize("sync", [False, True])
async def test_stats_returns_fresh_report_without_waiting_or_printing(sync, capsys):
reports = [
None,
"Function job: finished\nWorker utilization: unavailable\n",
"Function job: finished\nWorker utilization: 25.0%\n",
]
inner = SimpleNamespace(
stats=AsyncMock(side_effect=reports),
wait=AsyncMock(),
cancel=AsyncMock(),
)
job = AsyncJob(inner)
for expected in reports:
actual = await asyncio.to_thread(Job(job).stats) if sync else await job.stats()
assert actual == expected
inner.wait.assert_not_awaited()
inner.cancel.assert_not_awaited()
assert capsys.readouterr().out == ""
@pytest.mark.asyncio
@pytest.mark.parametrize("sync", [False, True])
async def test_stats_preserves_errors_and_rejects_untracked_jobs(sync):
error = RuntimeError("run reports are unavailable on this server")
job = AsyncJob(SimpleNamespace(stats=AsyncMock(side_effect=error)))
with pytest.raises(RuntimeError) as exc:
if sync:
await asyncio.to_thread(Job(job).stats)
else:
await job.stats()
assert exc.value is error
with pytest.raises(NotImplementedError, match="server-side Function job"):
if sync:
Job(None).stats()
else:
await AsyncJob(None).stats()
@pytest.mark.asyncio
@pytest.mark.parametrize("sync", [False, True])
async def test_export_timeline_writes_fresh_recording_without_waiting(sync, tmp_path):
+44
View File
@@ -2732,6 +2732,50 @@ async def test_remote_job_wait_disconnection_timeout_can_be_resumed(sync):
assert exc.value.__cause__.is_transport_error is True
@pytest.mark.asyncio
@pytest.mark.parametrize("sync", [False, True])
async def test_remote_job_stats_returns_native_optional_report(sync):
report = "Function job: finished\nWorker utilization: unavailable\n"
paths = []
def handler(request):
payload = json.loads(request.rfile.read(int(request.headers["Content-Length"])))
assert payload == {"job_id": "refresh-1"}
paths.append(request.path)
if request.path == "/v1/jobs/describe":
response = {
"job_id": "refresh-1",
"job_type": "function_refresh",
"job_state": "IN_PROGRESS",
"creation_ms": 1000,
}
elif request.path == "/v1/jobs/stats":
available = paths.count("/v1/jobs/stats") > 1
response = {
"job_id": "refresh-1",
"profile": {"future_field": True} if available else None,
"report": report if available else None,
}
else:
raise AssertionError(f"unexpected request: {request.path}")
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(json.dumps(response).encode())
if sync:
with mock_lancedb_connection(handler) as db:
job = db.open_job("refresh-1")
assert job.stats() is None
assert job.stats() == report
else:
async with mock_lancedb_connection_async(handler) as db:
job = await db.open_job("refresh-1")
assert await job.stats() is None
assert await job.stats() == report
assert paths == ["/v1/jobs/describe", "/v1/jobs/stats", "/v1/jobs/stats"]
@pytest.mark.asyncio
@pytest.mark.parametrize("sync", [False, True])
@pytest.mark.parametrize(
+5
View File
@@ -159,6 +159,11 @@ impl Job {
})
}
pub fn stats(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner.clone();
future_into_py(self_.py(), async move { inner.stats().await.infer_error() })
}
#[pyo3(signature = (*, limit=None, filter=None))]
pub fn events(
self_: PyRef<'_, Self>,