mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-13 14:16:54 +00:00
feat(python): export recorded Function job timelines (#4946)
Function jobs can now export their recorded execution timeline with
`job.export_timeline(path)` or `await job.export_timeline(path)`. The
SDK fetches a fresh Chrome Trace Event recording that opens in Perfetto,
including worker split spans and timestamped job phases. Batch counts
and latency statistics remain details rather than invented batch
intervals.
```python
job = db.open_job(job_id)
job.export_timeline("refresh.trace.json")
async_job = await async_db.open_job(job_id)
await async_job.export_timeline("refresh.trace.json")
```
The export does not wait for completion or cancel the job. Running jobs
return partial recordings. Local operations and SQL query handles remain
unsupported, and servers must support the timeline endpoint. The
destination is touched only after a successful fetch; network and
filesystem errors propagate. Rust freshness wrappers forward the
operation without changing the submitting table's read baseline.
This commit is contained in:
@@ -412,6 +412,7 @@ class Job:
|
||||
) -> Optional[str]: ...
|
||||
async def cancel(self) -> None: ...
|
||||
async def refresh(self) -> None: ...
|
||||
async def timeline(self) -> str: ...
|
||||
async def events(
|
||||
self,
|
||||
*,
|
||||
|
||||
@@ -7,6 +7,8 @@ import asyncio
|
||||
import reprlib
|
||||
import time
|
||||
from datetime import timedelta
|
||||
from os import PathLike
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Generic, Optional, TypeVar, cast
|
||||
|
||||
import pyarrow as pa
|
||||
@@ -422,6 +424,31 @@ class AsyncJob(Generic[T]):
|
||||
description = self._inner._description if self._inner is not None else None
|
||||
return getattr(description, name) if description is not None else None
|
||||
|
||||
async def export_timeline(self, path: str | PathLike[str]) -> None:
|
||||
"""Write a fresh recorded Function job timeline as Chrome trace JSON.
|
||||
|
||||
Open the file in Perfetto or Chrome tracing. The recording contains
|
||||
retained split spans grouped by worker and timestamped job phases.
|
||||
Batch counts and latency statistics are included as details; individual
|
||||
batch timestamps are not recorded. Running jobs export a partial snapshot
|
||||
without waiting or cancelling. Download again to include later work.
|
||||
|
||||
Requires a server-side Function job and a server supporting timeline
|
||||
export. Unsupported operations raise ``NotImplementedError`` or a server
|
||||
error. The destination is overwritten only after the trace is fetched.
|
||||
Filesystem errors propagate to the caller.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> await job.export_timeline("refresh.trace.json") # doctest: +SKIP
|
||||
"""
|
||||
if self._inner is None:
|
||||
raise NotImplementedError(
|
||||
"job timeline export requires a server-side Function job"
|
||||
)
|
||||
trace = await self._inner.timeline()
|
||||
await asyncio.to_thread(Path(path).write_text, trace, encoding="utf-8")
|
||||
|
||||
async def events(
|
||||
self,
|
||||
*,
|
||||
@@ -621,6 +648,22 @@ class Job(Generic[T]):
|
||||
def _field(self, name: str) -> Optional[Any]:
|
||||
return getattr(self._inner, name) if self._inner is not None else None
|
||||
|
||||
def export_timeline(self, path: str | PathLike[str]) -> None:
|
||||
"""Write Chrome trace JSON for this Function job.
|
||||
|
||||
See :meth:`AsyncJob.export_timeline`. Open the file in Perfetto or
|
||||
Chrome tracing.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> job.export_timeline("refresh.trace.json") # doctest: +SKIP
|
||||
"""
|
||||
if self._inner is None:
|
||||
raise NotImplementedError(
|
||||
"job timeline export requires a server-side Function job"
|
||||
)
|
||||
return LOOP.run(self._inner.export_timeline(path))
|
||||
|
||||
def events(
|
||||
self,
|
||||
*,
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import time
|
||||
from datetime import timedelta
|
||||
from types import SimpleNamespace
|
||||
@@ -751,3 +752,54 @@ async def test_query_wait_timeout_requires_keeping_original_handle():
|
||||
assert "continues on the server" in message
|
||||
assert "keep this handle to continue waiting" in message
|
||||
assert "db.open_job" not in message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("sync", [False, True])
|
||||
async def test_export_timeline_writes_fresh_recording_without_waiting(sync, tmp_path):
|
||||
trace = {
|
||||
"traceEvents": [{"ph": "X", "ts": 0, "dur": 1000}],
|
||||
"metadata": {"partial": True},
|
||||
}
|
||||
inner = SimpleNamespace(
|
||||
timeline=AsyncMock(return_value=json.dumps(trace)),
|
||||
wait=AsyncMock(),
|
||||
cancel=AsyncMock(),
|
||||
)
|
||||
job = AsyncJob(inner)
|
||||
path = tmp_path / "job.trace.json"
|
||||
if sync:
|
||||
await asyncio.to_thread(Job(job).export_timeline, str(path))
|
||||
else:
|
||||
await job.export_timeline(path)
|
||||
assert json.loads(path.read_text()) == trace
|
||||
inner.timeline.assert_awaited_once_with()
|
||||
inner.wait.assert_not_awaited()
|
||||
inner.cancel.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_export_timeline_failure_preserves_existing_file(tmp_path):
|
||||
path = tmp_path / "job.trace.json"
|
||||
path.write_text("previous recording")
|
||||
job = AsyncJob(
|
||||
SimpleNamespace(
|
||||
timeline=AsyncMock(side_effect=RuntimeError("unsupported server"))
|
||||
)
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="unsupported server"):
|
||||
await job.export_timeline(path)
|
||||
assert path.read_text() == "previous recording"
|
||||
with pytest.raises(NotImplementedError, match="server-side Function job"):
|
||||
await AsyncJob(None).export_timeline(path)
|
||||
with pytest.raises(NotImplementedError, match="server-side Function job"):
|
||||
Job(None).export_timeline(path)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_export_timeline_reports_filesystem_errors(tmp_path):
|
||||
job = AsyncJob(
|
||||
SimpleNamespace(timeline=AsyncMock(return_value='{"traceEvents":[]}'))
|
||||
)
|
||||
with pytest.raises(FileNotFoundError):
|
||||
await job.export_timeline(tmp_path / "missing" / "trace.json")
|
||||
|
||||
@@ -152,6 +152,13 @@ impl Job {
|
||||
self.inner.description().map(JobDescription::from)
|
||||
}
|
||||
|
||||
pub fn timeline(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.inner.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
Ok(inner.timeline().await.infer_error()?.to_string())
|
||||
})
|
||||
}
|
||||
|
||||
#[pyo3(signature = (*, limit=None, filter=None))]
|
||||
pub fn events(
|
||||
self_: PyRef<'_, Self>,
|
||||
|
||||
Reference in New Issue
Block a user