feat: connection-level job operations (#3755)

Adds job operations to the connection surface, building on the Job
handle from #3742: job(id), list_jobs, get_job, cancel_job, and
job_history, plus a non-blocking Job.status(). Implemented on the
Database trait (defaulting to NotSupported), the remote backend
(/v1/jobs), and the Python and Node bindings; job_history returns Arrow
batches.

errors() and progress() are not included.

Tested with mocked endpoints in all three languages.

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

python/src/connection.rs

@@ -13,7 +13,11 @@ use crate::{
     runtime::future_into_py,
     table::Table,
 };
-use arrow::{datatypes::Schema, ffi_stream::ArrowArrayStreamReader, pyarrow::FromPyArrow};
+use arrow::{
+    datatypes::Schema,
+    ffi_stream::ArrowArrayStreamReader,
+    pyarrow::{FromPyArrow, ToPyArrow},
+};
 use lancedb::{
     connection::Connection as LanceConnection,
     connection::NamespaceClientPushdownOperation,
@@ -24,7 +28,7 @@ use pyo3::{
     Bound, FromPyObject, Py, PyAny, PyRef, PyResult, Python,
     exceptions::{PyRuntimeError, PyValueError},
     pyclass, pyfunction, pymethods,
-    types::{PyDict, PyDictMethods},
+    types::{PyDict, PyDictMethods, PyList, PyListMethods},
 };
 
 #[pyclass]
@@ -536,6 +540,55 @@ impl Connection {
             })
         })
     }
+
+    pub fn job(&self, job_id: String) -> PyResult<crate::job::Job> {
+        let inner = self.get_inner()?.clone();
+        Ok(crate::job::Job::new(inner.job(job_id).infer_error()?))
+    }
+
+    pub fn list_jobs(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.get_inner()?.clone();
+        future_into_py(self_.py(), async move {
+            let jobs = inner.list_jobs().await.infer_error()?;
+            Ok(jobs
+                .into_iter()
+                .map(crate::job::JobInfo::from)
+                .collect::<Vec<_>>())
+        })
+    }
+
+    pub fn get_job(self_: PyRef<'_, Self>, job_id: String) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.get_inner()?.clone();
+        future_into_py(self_.py(), async move {
+            let description = inner.get_job(&job_id).await.infer_error()?;
+            Ok(description.map(crate::job::JobDescription::from))
+        })
+    }
+
+    pub fn cancel_job(self_: PyRef<'_, Self>, job_id: String) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.get_inner()?.clone();
+        future_into_py(self_.py(), async move {
+            inner.cancel_job(&job_id).await.infer_error()
+        })
+    }
+
+    #[pyo3(signature = (job_id=None))]
+    pub fn job_history(
+        self_: PyRef<'_, Self>,
+        job_id: Option<String>,
+    ) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.get_inner()?.clone();
+        future_into_py(self_.py(), async move {
+            let batches = inner.job_history(job_id.as_deref()).await.infer_error()?;
+            Python::attach(|py| {
+                let list = PyList::empty(py);
+                for batch in batches {
+                    list.append(batch.to_pyarrow(py)?)?;
+                }
+                Ok(list.unbind())
+            })
+        })
+    }
 }
 
 #[pyfunction]

python/src/job.rs

@@ -28,6 +28,14 @@ impl Job {
         self.inner.id().map(str::to_string)
     }
 
+    pub fn status(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner.clone();
+        future_into_py(
+            self_.py(),
+            async move { inner.status().await.infer_error() },
+        )
+    }
+
     pub fn wait(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.inner.clone();
         future_into_py(self_.py(), async move {
@@ -44,3 +52,94 @@ impl Job {
         })
     }
 }
+
+/// A row from `Connection.list_jobs`: one server-side job.
+#[pyclass(get_all, skip_from_py_object)]
+#[derive(Clone)]
+pub struct JobInfo {
+    job_id: String,
+    table: String,
+    job_type: String,
+    state: String,
+    created_at_millis: i64,
+}
+
+#[pymethods]
+impl JobInfo {
+    fn __repr__(&self) -> String {
+        format!(
+            "JobInfo(job_id={:?}, table={:?}, job_type={:?}, state={:?}, created_at_millis={})",
+            self.job_id, self.table, self.job_type, self.state, self.created_at_millis
+        )
+    }
+}
+
+impl From<lancedb::database::JobInfo> for JobInfo {
+    fn from(info: lancedb::database::JobInfo) -> Self {
+        Self {
+            job_id: info.job_id,
+            table: info.table,
+            job_type: info.job_type,
+            state: info.state,
+            created_at_millis: info.created_at_millis,
+        }
+    }
+}
+
+/// The server's account of why a job failed.
+#[pyclass(get_all, skip_from_py_object)]
+#[derive(Clone)]
+pub struct JobFailureInfo {
+    phase: Option<String>,
+    message: Option<String>,
+    retryable: Option<bool>,
+}
+
+#[pymethods]
+impl JobFailureInfo {
+    fn __repr__(&self) -> String {
+        format!(
+            "JobFailureInfo(phase={:?}, message={:?}, retryable={:?})",
+            self.phase, self.message, self.retryable
+        )
+    }
+}
+
+/// A described job from `Connection.get_job`.
+#[pyclass(get_all, skip_from_py_object)]
+#[derive(Clone)]
+pub struct JobDescription {
+    job_id: String,
+    job_type: String,
+    state: String,
+    creation_ms: i64,
+    spec_json: Option<String>,
+    failure: Option<JobFailureInfo>,
+}
+
+#[pymethods]
+impl JobDescription {
+    fn __repr__(&self) -> String {
+        format!(
+            "JobDescription(job_id={:?}, job_type={:?}, state={:?}, creation_ms={})",
+            self.job_id, self.job_type, self.state, self.creation_ms
+        )
+    }
+}
+
+impl From<lancedb::database::JobDescription> for JobDescription {
+    fn from(description: lancedb::database::JobDescription) -> Self {
+        Self {
+            job_id: description.job_id,
+            job_type: description.job_type,
+            state: description.state,
+            creation_ms: description.creation_ms,
+            spec_json: (!description.spec.is_null()).then(|| description.spec.to_string()),
+            failure: description.failure.map(|failure| JobFailureInfo {
+                phase: failure.phase,
+                message: failure.message,
+                retryable: failure.retryable,
+            }),
+        }
+    }
+}

python/src/lib.rs

@@ -46,6 +46,9 @@ pub fn _lancedb(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_class::<Session>()?;
     m.add_class::<Table>()?;
     m.add_class::<crate::job::Job>()?;
+    m.add_class::<crate::job::JobInfo>()?;
+    m.add_class::<crate::job::JobDescription>()?;
+    m.add_class::<crate::job::JobFailureInfo>()?;
     m.add_class::<PyBlobFile>()?;
     m.add_class::<IndexConfig>()?;
     m.add_class::<Query>()?;