mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-16 07:36:50 +00:00
Expose `Job::stats()` in Rust and synchronous/asynchronous Python jobs. It reads `POST /v1/jobs/stats` and returns the canonical server-rendered report verbatim, or `None` for a running job or one without a readable durable profile. It does not wait for completion or recompute measurements. On `cc25005525671756b5cf5ebd96492b792e9aa1cb`, all three Rust transport tests and six Python tests passed against the matching rebuilt native extension; strict SDK Clippy also passed. These tests cover optional reports, exact text preservation, fresh reads, errors, and the HTTP boundary with a controlled server. Against the actual Phalanx HTTP route from [Sophon #10247](https://github.com/lancedb/sophon/pull/10247) (`5be3236`), the same native extension also completed two sync and two async reads after durable registry archival, preserving the exact canonical report. This is an HTTP/registry integration fixture, not a Function workload or shared-cluster acceptance. The private runner's JSON field-order collector failed after the successful probe; all raw evidence was retained and hash-verified, and the original receipt was recovered with the corrected parser without rerunning the probe. The follow-up `ad36f66` only corrects the synchronous Python API-reference link to the asynchronous method. Ruff checks and formatting passed; runtime evidence above remains applicable. The actual Phalanx/native-SDK probe covers archived reports; running-job null availability is covered by separate backend and SDK transport tests, not that end-to-end probe. Part of ENT-3501.
372 lines
12 KiB
Rust
372 lines
12 KiB
Rust
// SPDX-License-Identifier: Apache-2.0
|
|
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
|
|
|
use std::sync::Arc;
|
|
|
|
use crate::runtime::future_into_py;
|
|
use arrow::{
|
|
datatypes::Schema,
|
|
pyarrow::{IntoPyArrow, Table as PyArrowTable},
|
|
};
|
|
use lancedb::job::JobEventsRequest;
|
|
use pyo3::{
|
|
Bound, Py, PyAny, PyRef, PyResult, Python,
|
|
exceptions::PyValueError,
|
|
pyclass, pymethods,
|
|
types::{PyAnyMethods, PyDict, PyDictMethods},
|
|
};
|
|
use serde::Serialize;
|
|
|
|
use crate::error::PythonErrorExt;
|
|
|
|
const REPR_INDENT: &str = " ";
|
|
|
|
/// Parse a stored JSON payload into Python data. The bindings carry these as
|
|
/// strings because that is what crosses the boundary cheaply; the public
|
|
/// Python surface is the parsed form.
|
|
fn parse_json_payload<'py>(
|
|
py: Python<'py>,
|
|
raw: Option<&str>,
|
|
) -> PyResult<Option<Bound<'py, PyAny>>> {
|
|
match raw {
|
|
None => Ok(None),
|
|
Some(raw) => Ok(Some(py.import("json")?.call_method1("loads", (raw,))?)),
|
|
}
|
|
}
|
|
|
|
/// A payload rendered as indented JSON, aligned under the field that holds it.
|
|
fn pretty_json_payload(py: Python<'_>, raw: Option<&str>) -> PyResult<Option<String>> {
|
|
let Some(parsed) = parse_json_payload(py, raw)? else {
|
|
return Ok(None);
|
|
};
|
|
let kwargs = PyDict::new(py);
|
|
kwargs.set_item("indent", 4)?;
|
|
let rendered: String = py
|
|
.import("json")?
|
|
.call_method("dumps", (parsed,), Some(&kwargs))?
|
|
.extract()?;
|
|
Ok(Some(rendered.replace('\n', &format!("\n{REPR_INDENT}"))))
|
|
}
|
|
|
|
#[pyclass]
|
|
pub struct Job {
|
|
inner: Arc<lancedb::Job<std::result::Result<Option<String>, String>>>,
|
|
}
|
|
|
|
impl Job {
|
|
pub(crate) fn new(inner: lancedb::Job) -> Self {
|
|
Self {
|
|
inner: Arc::new(inner.map(|()| Ok(None))),
|
|
}
|
|
}
|
|
|
|
pub(crate) fn new_typed<T>(inner: lancedb::Job<T>) -> Self
|
|
where
|
|
T: Clone + Serialize + Send + Sync + 'static,
|
|
{
|
|
Self {
|
|
inner: Arc::new(inner.map(|result| {
|
|
serde_json::to_string(&result)
|
|
.map(Some)
|
|
.map_err(|error| format!("failed to serialize typed job result: {error}"))
|
|
})),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[pymethods]
|
|
impl Job {
|
|
#[getter]
|
|
pub fn id(&self) -> Option<String> {
|
|
self.inner.id().map(str::to_string)
|
|
}
|
|
|
|
#[getter]
|
|
pub fn job_type(&self) -> Option<String> {
|
|
self.inner.job_type()
|
|
}
|
|
|
|
pub fn status(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
|
let inner = self_.inner.clone();
|
|
future_into_py(
|
|
self_.py(),
|
|
async move { inner.status().await.infer_error() },
|
|
)
|
|
}
|
|
|
|
#[pyo3(signature = (progress=None))]
|
|
pub fn wait(self_: PyRef<'_, Self>, progress: Option<Py<PyAny>>) -> PyResult<Bound<'_, PyAny>> {
|
|
let inner = self_.inner.clone();
|
|
future_into_py(self_.py(), async move {
|
|
let result = if let Some(progress) = progress {
|
|
let observer = move |description: &lancedb::database::JobDescription| {
|
|
Python::attach(|py| {
|
|
let snapshot = description.progress_snapshot();
|
|
let callback_result: PyResult<()> = (|| {
|
|
let raw = snapshot.to_string();
|
|
let parsed = parse_json_payload(py, Some(&raw))?
|
|
.ok_or_else(|| PyValueError::new_err("missing progress"))?;
|
|
progress.bind(py).call1((parsed,))?;
|
|
Ok(())
|
|
})();
|
|
if let Err(error) = callback_result {
|
|
log::warn!("job progress callback error: {error}");
|
|
}
|
|
});
|
|
};
|
|
inner.wait_with_progress(observer).await.infer_error()?
|
|
} else {
|
|
inner.wait().await.infer_error()?
|
|
};
|
|
result
|
|
.map_err(|message| lancedb::Error::Runtime { message })
|
|
.infer_error()
|
|
})
|
|
}
|
|
|
|
pub fn cancel(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
|
let inner = self_.inner.clone();
|
|
future_into_py(self_.py(), async move {
|
|
inner.cancel().await.infer_error()?;
|
|
Ok(())
|
|
})
|
|
}
|
|
|
|
pub fn refresh(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
|
let inner = self_.inner.clone();
|
|
future_into_py(self_.py(), async move {
|
|
inner.refresh().await.infer_error()?;
|
|
Ok(())
|
|
})
|
|
}
|
|
|
|
/// The last observed lifecycle state, without contacting the backend.
|
|
#[getter]
|
|
pub fn _state(&self) -> Option<String> {
|
|
self.inner.state()
|
|
}
|
|
|
|
/// The last observed server-side record. `None` for an in-process job.
|
|
#[getter]
|
|
pub fn _description(&self) -> Option<JobDescription> {
|
|
self.inner.description().map(JobDescription::from)
|
|
}
|
|
|
|
pub fn timeline(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
|
let inner = self_.inner.clone();
|
|
future_into_py(self_.py(), async move {
|
|
Ok(inner.timeline().await.infer_error()?.to_string())
|
|
})
|
|
}
|
|
|
|
pub fn stats(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
|
let inner = self_.inner.clone();
|
|
future_into_py(self_.py(), async move { inner.stats().await.infer_error() })
|
|
}
|
|
|
|
#[pyo3(signature = (*, limit=None, filter=None))]
|
|
pub fn events(
|
|
self_: PyRef<'_, Self>,
|
|
limit: Option<u32>,
|
|
filter: Option<String>,
|
|
) -> PyResult<Bound<'_, PyAny>> {
|
|
let inner = self_.inner.clone();
|
|
let request = JobEventsRequest { limit, filter };
|
|
future_into_py(self_.py(), async move {
|
|
let batches = inner.events(request).await.infer_error()?;
|
|
Python::attach(|py| {
|
|
let schema = batches
|
|
.first()
|
|
.map(|batch| batch.schema())
|
|
.unwrap_or_else(|| Arc::new(Schema::empty()));
|
|
let table = PyArrowTable::try_new(batches, schema)
|
|
.map_err(|err| PyValueError::new_err(err.to_string()))?;
|
|
table.into_pyarrow(py).map(|table| table.unbind())
|
|
})
|
|
})
|
|
}
|
|
}
|
|
|
|
/// A row from `Connection.list_jobs`: one server-side job.
|
|
#[pyclass(module = "lancedb._lancedb", get_all, skip_from_py_object)]
|
|
#[derive(Clone)]
|
|
pub struct JobInfo {
|
|
job_id: String,
|
|
table: String,
|
|
job_type: String,
|
|
state: String,
|
|
created_at_millis: i64,
|
|
_progress_json: Option<String>,
|
|
paused: bool,
|
|
}
|
|
|
|
#[pymethods]
|
|
impl JobInfo {
|
|
fn __repr__(&self) -> String {
|
|
format!(
|
|
"JobInfo(job_id={:?}, table={:?}, job_type={:?}, state={:?}, created_at_millis={})",
|
|
self.job_id, self.table, self.job_type, self.state, self.created_at_millis
|
|
)
|
|
}
|
|
|
|
/// Structured progress for job types that report it. Missing fields are
|
|
/// unknown, not zero.
|
|
#[getter]
|
|
fn progress<'py>(&self, py: Python<'py>) -> PyResult<Option<Bound<'py, PyAny>>> {
|
|
parse_json_payload(py, self._progress_json.as_deref())
|
|
}
|
|
|
|
/// Whether an operator has parked this job.
|
|
#[getter]
|
|
fn paused(&self) -> bool {
|
|
self.paused
|
|
}
|
|
}
|
|
|
|
impl From<lancedb::database::JobInfo> for JobInfo {
|
|
fn from(info: lancedb::database::JobInfo) -> Self {
|
|
let progress = info.progress_snapshot().to_string();
|
|
Self {
|
|
job_id: info.job_id,
|
|
table: info.table,
|
|
job_type: info.job_type,
|
|
state: info.state,
|
|
created_at_millis: info.created_at_millis,
|
|
_progress_json: Some(progress),
|
|
paused: info.paused,
|
|
}
|
|
}
|
|
}
|
|
|
|
/// The server's account of why a job failed.
|
|
#[pyclass(module = "lancedb._lancedb", get_all, skip_from_py_object)]
|
|
#[derive(Clone, Debug)]
|
|
pub struct JobFailureInfo {
|
|
phase: Option<String>,
|
|
message: Option<String>,
|
|
/// User-code traceback, possibly truncated by the server.
|
|
traceback: Option<String>,
|
|
retryable: Option<bool>,
|
|
sql_job_id: Option<String>,
|
|
}
|
|
|
|
impl From<lancedb::error::JobFailure> for JobFailureInfo {
|
|
fn from(failure: lancedb::error::JobFailure) -> Self {
|
|
Self {
|
|
phase: failure.phase,
|
|
message: failure.message,
|
|
traceback: failure.traceback,
|
|
retryable: failure.retryable,
|
|
sql_job_id: failure.sql_job_id,
|
|
}
|
|
}
|
|
}
|
|
|
|
#[pymethods]
|
|
impl JobFailureInfo {
|
|
fn __repr__(&self) -> String {
|
|
let phase = self
|
|
.phase
|
|
.as_ref()
|
|
.map_or_else(|| "None".to_string(), |value| format!("{value:?}"));
|
|
let message = self
|
|
.message
|
|
.as_ref()
|
|
.map_or_else(|| "None".to_string(), |value| format!("{value:?}"));
|
|
let retryable = match self.retryable {
|
|
Some(true) => "True",
|
|
Some(false) => "False",
|
|
None => "None",
|
|
};
|
|
let sql_job_id = self
|
|
.sql_job_id
|
|
.as_ref()
|
|
.map_or_else(|| "None".to_string(), |value| format!("{value:?}"));
|
|
format!(
|
|
"JobFailureInfo(phase={phase}, message={message}, retryable={retryable}, \
|
|
sql_job_id={sql_job_id})"
|
|
)
|
|
}
|
|
}
|
|
|
|
/// The server-side record behind a `Job` handle.
|
|
#[pyclass(module = "lancedb._lancedb", get_all, skip_from_py_object)]
|
|
#[derive(Clone)]
|
|
pub struct JobDescription {
|
|
job_id: String,
|
|
job_type: String,
|
|
state: String,
|
|
creation_ms: i64,
|
|
/// Internal: the wire form behind the `spec` property.
|
|
_spec_json: Option<String>,
|
|
/// Internal: the wire form behind the `result` property.
|
|
_result_json: Option<String>,
|
|
_progress_json: Option<String>,
|
|
failure: Option<JobFailureInfo>,
|
|
}
|
|
|
|
#[pymethods]
|
|
impl JobDescription {
|
|
/// The job-type-specific specification it was submitted with.
|
|
#[getter]
|
|
fn spec<'py>(&self, py: Python<'py>) -> PyResult<Option<Bound<'py, PyAny>>> {
|
|
parse_json_payload(py, self._spec_json.as_deref())
|
|
}
|
|
|
|
/// The job-type-specific terminal result. `None` until the job succeeds.
|
|
#[getter]
|
|
fn result<'py>(&self, py: Python<'py>) -> PyResult<Option<Bound<'py, PyAny>>> {
|
|
parse_json_payload(py, self._result_json.as_deref())
|
|
}
|
|
|
|
/// Structured progress snapshot. Lifecycle and queue fields are present
|
|
/// even when a job type or older server supplies no counters.
|
|
#[getter]
|
|
fn progress<'py>(&self, py: Python<'py>) -> PyResult<Option<Bound<'py, PyAny>>> {
|
|
parse_json_payload(py, self._progress_json.as_deref())
|
|
}
|
|
|
|
fn __repr__(&self, py: Python<'_>) -> PyResult<String> {
|
|
let mut fields = vec![
|
|
format!("job_id={:?}", self.job_id),
|
|
format!("job_type={:?}", self.job_type),
|
|
format!("state={:?}", self.state),
|
|
format!("creation_ms={}", self.creation_ms),
|
|
];
|
|
// Lay the payloads out as indented JSON, the same way the `Job` repr
|
|
// does, so the two agree on how the same data looks.
|
|
for (name, payload) in [("spec", &self._spec_json), ("result", &self._result_json)] {
|
|
if let Some(rendered) = pretty_json_payload(py, payload.as_deref())? {
|
|
fields.push(format!("{name}={rendered}"));
|
|
}
|
|
}
|
|
if let Some(failure) = &self.failure {
|
|
fields.push(format!("failure={}", failure.__repr__()));
|
|
}
|
|
let body = fields
|
|
.iter()
|
|
.map(|field| format!("\n{REPR_INDENT}{field},"))
|
|
.collect::<String>();
|
|
Ok(format!("JobDescription({body}\n)"))
|
|
}
|
|
}
|
|
|
|
impl From<lancedb::database::JobDescription> for JobDescription {
|
|
fn from(description: lancedb::database::JobDescription) -> Self {
|
|
let progress_json = description.progress_snapshot().to_string();
|
|
Self {
|
|
job_id: description.job_id,
|
|
job_type: description.job_type,
|
|
state: description.state,
|
|
creation_ms: description.creation_ms,
|
|
_spec_json: (!description.spec.is_null()).then(|| description.spec.to_string()),
|
|
_result_json: description
|
|
.result
|
|
.filter(|result| !result.is_null())
|
|
.map(|result| result.to_string()),
|
|
_progress_json: Some(progress_json),
|
|
failure: description.failure.map(Into::into),
|
|
}
|
|
}
|
|
}
|