feat(python): export recorded Function job timelines (#4946)

Function jobs can now export their recorded execution timeline with
`job.export_timeline(path)` or `await job.export_timeline(path)`. The
SDK fetches a fresh Chrome Trace Event recording that opens in Perfetto,
including worker split spans and timestamped job phases. Batch counts
and latency statistics remain details rather than invented batch
intervals.

```python
job = db.open_job(job_id)
job.export_timeline("refresh.trace.json")

async_job = await async_db.open_job(job_id)
await async_job.export_timeline("refresh.trace.json")
```

The export does not wait for completion or cancel the job. Running jobs
return partial recordings. Local operations and SQL query handles remain
unsupported, and servers must support the timeline endpoint. The
destination is touched only after a successful fetch; network and
filesystem errors propagate. Rust freshness wrappers forward the
operation without changing the submitting table's read baseline.
This commit is contained in:
Jack Ye
2026-11-11 07:35:43 -08:00
committed by GitHub
parent e9774ed168
commit ea1cf17f89
8 changed files with 210 additions and 0 deletions
+1
View File
@@ -412,6 +412,7 @@ class Job:
) -> Optional[str]: ...
async def cancel(self) -> None: ...
async def refresh(self) -> None: ...
async def timeline(self) -> str: ...
async def events(
self,
*,
+43
View File
@@ -7,6 +7,8 @@ import asyncio
import reprlib
import time
from datetime import timedelta
from os import PathLike
from pathlib import Path
from typing import Any, Callable, Generic, Optional, TypeVar, cast
import pyarrow as pa
@@ -422,6 +424,31 @@ class AsyncJob(Generic[T]):
description = self._inner._description if self._inner is not None else None
return getattr(description, name) if description is not None else None
async def export_timeline(self, path: str | PathLike[str]) -> None:
"""Write a fresh recorded Function job timeline as Chrome trace JSON.
Open the file in Perfetto or Chrome tracing. The recording contains
retained split spans grouped by worker and timestamped job phases.
Batch counts and latency statistics are included as details; individual
batch timestamps are not recorded. Running jobs export a partial snapshot
without waiting or cancelling. Download again to include later work.
Requires a server-side Function job and a server supporting timeline
export. Unsupported operations raise ``NotImplementedError`` or a server
error. The destination is overwritten only after the trace is fetched.
Filesystem errors propagate to the caller.
Examples
--------
>>> await job.export_timeline("refresh.trace.json") # doctest: +SKIP
"""
if self._inner is None:
raise NotImplementedError(
"job timeline export requires a server-side Function job"
)
trace = await self._inner.timeline()
await asyncio.to_thread(Path(path).write_text, trace, encoding="utf-8")
async def events(
self,
*,
@@ -621,6 +648,22 @@ class Job(Generic[T]):
def _field(self, name: str) -> Optional[Any]:
return getattr(self._inner, name) if self._inner is not None else None
def export_timeline(self, path: str | PathLike[str]) -> None:
"""Write Chrome trace JSON for this Function job.
See :meth:`AsyncJob.export_timeline`. Open the file in Perfetto or
Chrome tracing.
Examples
--------
>>> job.export_timeline("refresh.trace.json") # doctest: +SKIP
"""
if self._inner is None:
raise NotImplementedError(
"job timeline export requires a server-side Function job"
)
return LOOP.run(self._inner.export_timeline(path))
def events(
self,
*,
+52
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import asyncio
import json
import time
from datetime import timedelta
from types import SimpleNamespace
@@ -751,3 +752,54 @@ async def test_query_wait_timeout_requires_keeping_original_handle():
assert "continues on the server" in message
assert "keep this handle to continue waiting" in message
assert "db.open_job" not in message
@pytest.mark.asyncio
@pytest.mark.parametrize("sync", [False, True])
async def test_export_timeline_writes_fresh_recording_without_waiting(sync, tmp_path):
trace = {
"traceEvents": [{"ph": "X", "ts": 0, "dur": 1000}],
"metadata": {"partial": True},
}
inner = SimpleNamespace(
timeline=AsyncMock(return_value=json.dumps(trace)),
wait=AsyncMock(),
cancel=AsyncMock(),
)
job = AsyncJob(inner)
path = tmp_path / "job.trace.json"
if sync:
await asyncio.to_thread(Job(job).export_timeline, str(path))
else:
await job.export_timeline(path)
assert json.loads(path.read_text()) == trace
inner.timeline.assert_awaited_once_with()
inner.wait.assert_not_awaited()
inner.cancel.assert_not_awaited()
@pytest.mark.asyncio
async def test_export_timeline_failure_preserves_existing_file(tmp_path):
path = tmp_path / "job.trace.json"
path.write_text("previous recording")
job = AsyncJob(
SimpleNamespace(
timeline=AsyncMock(side_effect=RuntimeError("unsupported server"))
)
)
with pytest.raises(RuntimeError, match="unsupported server"):
await job.export_timeline(path)
assert path.read_text() == "previous recording"
with pytest.raises(NotImplementedError, match="server-side Function job"):
await AsyncJob(None).export_timeline(path)
with pytest.raises(NotImplementedError, match="server-side Function job"):
Job(None).export_timeline(path)
@pytest.mark.asyncio
async def test_export_timeline_reports_filesystem_errors(tmp_path):
job = AsyncJob(
SimpleNamespace(timeline=AsyncMock(return_value='{"traceEvents":[]}'))
)
with pytest.raises(FileNotFoundError):
await job.export_timeline(tmp_path / "missing" / "trace.json")
+7
View File
@@ -152,6 +152,13 @@ impl Job {
self.inner.description().map(JobDescription::from)
}
pub fn timeline(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
Ok(inner.timeline().await.infer_error()?.to_string())
})
}
#[pyo3(signature = (*, limit=None, filter=None))]
pub fn events(
self_: PyRef<'_, Self>,