feat: connection-level job operations (#3755)

Adds job operations to the connection surface, building on the Job
handle from #3742: job(id), list_jobs, get_job, cancel_job, and
job_history, plus a non-blocking Job.status(). Implemented on the
Database trait (defaulting to NotSupported), the remote backend
(/v1/jobs), and the Python and Node bindings; job_history returns Arrow
batches.

errors() and progress() are not included.

Tested with mocked endpoints in all three languages.

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

python/python/lancedb/_lancedb.pyi

@@ -146,6 +146,13 @@ class Connection(object):
         start_after: Optional[str],
         limit: Optional[int],
     ) -> list[str]: ...  # Deprecated: Use list_tables instead
+    def job(self, job_id: str) -> Job: ...
+    async def list_jobs(self) -> List[JobInfo]: ...
+    async def get_job(self, job_id: str) -> Optional[JobDescription]: ...
+    async def cancel_job(self, job_id: str) -> bool: ...
+    async def job_history(
+        self, job_id: Optional[str] = None
+    ) -> List[pa.RecordBatch]: ...
     async def create_table(
         self,
         name: str,
@@ -212,9 +219,44 @@ class BlobFile:
 class Job:
     @property
     def id(self) -> Optional[str]: ...
+    async def status(self) -> str: ...
     async def wait(self) -> None: ...
     async def cancel(self) -> None: ...
 
+class JobInfo:
+    @property
+    def job_id(self) -> str: ...
+    @property
+    def table(self) -> str: ...
+    @property
+    def job_type(self) -> str: ...
+    @property
+    def state(self) -> str: ...
+    @property
+    def created_at_millis(self) -> int: ...
+
+class JobFailureInfo:
+    @property
+    def phase(self) -> Optional[str]: ...
+    @property
+    def message(self) -> Optional[str]: ...
+    @property
+    def retryable(self) -> Optional[bool]: ...
+
+class JobDescription:
+    @property
+    def job_id(self) -> str: ...
+    @property
+    def job_type(self) -> str: ...
+    @property
+    def state(self) -> str: ...
+    @property
+    def creation_ms(self) -> int: ...
+    @property
+    def spec_json(self) -> Optional[str]: ...
+    @property
+    def failure(self) -> Optional[JobFailureInfo]: ...
+
 class Table:
     def name(self) -> str: ...
     def __repr__(self) -> str: ...

python/python/lancedb/db.py

@@ -45,6 +45,7 @@ from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError
 
 from . import __version__
 from ._lancedb import connect as lancedb_connect  # type: ignore
+from .job import AsyncJob, Job
 from .table import (
     AsyncTable,
     LanceTable,
@@ -63,6 +64,7 @@ if TYPE_CHECKING:
     from .pydantic import LanceModel
 
     from ._lancedb import Connection as LanceDbConnection
+    from ._lancedb import JobDescription, JobInfo
     from .common import DATA, URI
     from .embeddings import EmbeddingFunctionConfig
     from ._lancedb import Session
@@ -608,6 +610,46 @@ class DBConnection(EnforceOverrides):
         """
         raise NotImplementedError("serialize is not supported for this connection type")
 
+    def job(self, job_id: str) -> Job:
+        """A [Job][lancedb.job.Job] handle for a server-side job by id.
+
+        The handle is constructed without a server round trip; an unknown id
+        surfaces when the handle is used. Dropping the handle has no effect
+        on the job itself.
+        """
+        raise NotImplementedError("job is not supported for this connection type")
+
+    def list_jobs(self) -> List[JobInfo]:
+        """List server-side jobs across the database's tables."""
+        raise NotImplementedError("list_jobs is not supported for this connection type")
+
+    def get_job(self, job_id: str) -> Optional[JobDescription]:
+        """Describe a single server-side job by id.
+
+        Returns None when the server has no such job.
+        """
+        raise NotImplementedError("get_job is not supported for this connection type")
+
+    def cancel_job(self, job_id: str) -> bool:
+        """Request cancellation of a server-side job by id.
+
+        Returns True if the server accepted the cancellation, False if no
+        such job exists. Cancelling an already-terminal job is a no-op
+        success.
+        """
+        raise NotImplementedError(
+            "cancel_job is not supported for this connection type"
+        )
+
+    def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
+        """The lifecycle event history of a server-side job, as Arrow batches.
+
+        Lists history across all jobs when `job_id` is None.
+        """
+        raise NotImplementedError(
+            "job_history is not supported for this connection type"
+        )
+
 
 class LanceDBConnection(DBConnection):
     """
@@ -1170,6 +1212,47 @@ class LanceDBConnection(DBConnection):
             )
         )
 
+    @override
+    def job(self, job_id: str) -> Job:
+        """A [Job][lancedb.job.Job] handle for a server-side job by id.
+
+        The handle is constructed without a server round trip; an unknown id
+        surfaces when the handle is used. Dropping the handle has no effect
+        on the job itself.
+        """
+        return Job(self._conn.job(job_id))
+
+    @override
+    def list_jobs(self) -> List[JobInfo]:
+        """List server-side jobs across the database's tables."""
+        return LOOP.run(self._conn.list_jobs())
+
+    @override
+    def get_job(self, job_id: str) -> Optional[JobDescription]:
+        """Describe a single server-side job by id.
+
+        Returns None when the server has no such job.
+        """
+        return LOOP.run(self._conn.get_job(job_id))
+
+    @override
+    def cancel_job(self, job_id: str) -> bool:
+        """Request cancellation of a server-side job by id.
+
+        Returns True if the server accepted the cancellation, False if no
+        such job exists. Cancelling an already-terminal job is a no-op
+        success.
+        """
+        return LOOP.run(self._conn.cancel_job(job_id))
+
+    @override
+    def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
+        """The lifecycle event history of a server-side job, as Arrow batches.
+
+        Lists history across all jobs when `job_id` is None.
+        """
+        return LOOP.run(self._conn.job_history(job_id))
+
     @override
     def namespace_client(self) -> LanceNamespace:
         """Get the equivalent namespace client for this connection.
@@ -1879,6 +1962,43 @@ class AsyncConnection(object):
             namespace_path = []
         await self._inner.drop_all_tables(namespace_path=namespace_path)
 
+    def job(self, job_id: str) -> AsyncJob:
+        """An [AsyncJob][lancedb.job.AsyncJob] handle for a server-side job
+        by id.
+
+        The handle is constructed without a server round trip; an unknown id
+        surfaces when the handle is used. Dropping the handle has no effect
+        on the job itself.
+        """
+        return AsyncJob(self._inner.job(job_id))
+
+    async def list_jobs(self) -> List[JobInfo]:
+        """List server-side jobs across the database's tables."""
+        return await self._inner.list_jobs()
+
+    async def get_job(self, job_id: str) -> Optional[JobDescription]:
+        """Describe a single server-side job by id.
+
+        Returns None when the server has no such job.
+        """
+        return await self._inner.get_job(job_id)
+
+    async def cancel_job(self, job_id: str) -> bool:
+        """Request cancellation of a server-side job by id.
+
+        Returns True if the server accepted the cancellation, False if no
+        such job exists. Cancelling an already-terminal job is a no-op
+        success.
+        """
+        return await self._inner.cancel_job(job_id)
+
+    async def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
+        """The lifecycle event history of a server-side job, as Arrow batches.
+
+        Lists history across all jobs when `job_id` is None.
+        """
+        return await self._inner.job_history(job_id)
+
     async def namespace_client(self) -> LanceNamespace:
         """Get the equivalent namespace client for this connection.
 

python/python/lancedb/job.py

@@ -32,6 +32,18 @@ class AsyncJob:
         """
         return self._inner.id if self._inner is not None else None
 
+    async def status(self) -> str:
+        """The operation's current lifecycle state: "running", "finished",
+        "failed", or "cancelled".
+
+        A point snapshot; unlike `wait` it does not block or raise on a
+        terminal failure state. States a newer server reports that this
+        client version does not know pass through as-is.
+        """
+        if self._inner is None:
+            return "finished"
+        return await self._inner.status()
+
     async def wait(self, timeout: Optional[timedelta] = None):
         """Wait until the operation reaches a terminal state.
 
@@ -66,6 +78,16 @@ class Job:
         """
         return self._inner.id if self._inner is not None else None
 
+    def status(self) -> str:
+        """The operation's current lifecycle state: "running", "finished",
+        "failed", or "cancelled".
+
+        See :meth:`AsyncJob.status`.
+        """
+        if self._inner is None:
+            return "finished"
+        return LOOP.run(self._inner.status())
+
     def wait(self, timeout: Optional[timedelta] = None):
         """Block until the operation reaches a terminal state.
 

python/python/lancedb/remote/db.py

@@ -7,7 +7,7 @@ import json
 import logging
 from concurrent.futures import ThreadPoolExecutor
 import sys
-from typing import Any, Dict, Iterable, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
 from urllib.parse import urlparse
 import warnings
 
@@ -23,6 +23,10 @@ import pyarrow as pa
 
 from ..common import DATA
 from ..db import DBConnection, LOOP
+from ..job import Job
+
+if TYPE_CHECKING:
+    from .._lancedb import JobDescription, JobInfo
 from ..embeddings import EmbeddingFunctionConfig
 from lance_namespace import (
     LanceNamespace,
@@ -689,6 +693,47 @@ class RemoteDBConnection(DBConnection):
             )
         )
 
+    @override
+    def job(self, job_id: str) -> Job:
+        """A [Job][lancedb.job.Job] handle for a server-side job by id.
+
+        The handle is constructed without a server round trip; an unknown id
+        surfaces when the handle is used. Dropping the handle has no effect
+        on the job itself.
+        """
+        return Job(self._conn.job(job_id))
+
+    @override
+    def list_jobs(self) -> List["JobInfo"]:
+        """List server-side jobs across the database's tables."""
+        return LOOP.run(self._conn.list_jobs())
+
+    @override
+    def get_job(self, job_id: str) -> Optional["JobDescription"]:
+        """Describe a single server-side job by id.
+
+        Returns None when the server has no such job.
+        """
+        return LOOP.run(self._conn.get_job(job_id))
+
+    @override
+    def cancel_job(self, job_id: str) -> bool:
+        """Request cancellation of a server-side job by id.
+
+        Returns True if the server accepted the cancellation, False if no
+        such job exists. Cancelling an already-terminal job is a no-op
+        success.
+        """
+        return LOOP.run(self._conn.cancel_job(job_id))
+
+    @override
+    def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
+        """The lifecycle event history of a server-side job, as Arrow batches.
+
+        Lists history across all jobs when `job_id` is None.
+        """
+        return LOOP.run(self._conn.job_history(job_id))
+
     @override
     def namespace_client(self) -> LanceNamespace:
         """Get the equivalent namespace client for this connection.

python/python/tests/test_remote_db.py

@@ -2153,3 +2153,119 @@ def test_remote_blob_byte_apis_not_supported_on_old_server():
             table.fetch_blobs("image", [1])
         with pytest.raises(NotImplementedError, match="not supported"):
             table.fetch_blob_files("image", [1])
+
+
+def test_remote_connection_jobs_surface():
+    from lancedb.exceptions import JobFailedError
+
+    schema = pa.schema([("state", pa.string())])
+    batch = pa.record_batch([pa.array(["created", "done"])], schema=schema)
+    sink = pa.BufferOutputStream()
+    with pa.ipc.new_stream(sink, schema) as writer:
+        writer.write_batch(batch)
+    events_body = sink.getvalue().to_pybytes()
+
+    def handler(request):
+        content_len = int(request.headers.get("Content-Length", 0))
+        body = request.rfile.read(content_len) if content_len > 0 else b""
+        payload = json.loads(body) if body else {}
+        if request.path == "/v1/jobs/list":
+            if payload.get("page_token") is None:
+                rsp = dict(
+                    jobs=[
+                        dict(
+                            job_id="job-1",
+                            table="t1",
+                            job_type="create_index",
+                            state="in_progress",
+                            created_at_millis=1000,
+                        )
+                    ],
+                    page_token="next",
+                )
+            else:
+                assert payload["page_token"] == "next"
+                rsp = dict(
+                    jobs=[
+                        dict(
+                            job_id="job-2",
+                            table="t2",
+                            job_type="create_index",
+                            state="succeeded",
+                            created_at_millis=2000,
+                        )
+                    ]
+                )
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            request.wfile.write(json.dumps(rsp).encode())
+        elif request.path == "/v1/jobs/describe":
+            if payload["job_id"] != "job-1":
+                request.send_response(404)
+                request.end_headers()
+                return
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            request.wfile.write(
+                json.dumps(
+                    dict(
+                        job_id="job-1",
+                        job_type="create_index",
+                        job_state="FAILED",
+                        creation_ms=1000,
+                        spec=dict(column="vec"),
+                        failure=dict(
+                            phase="execute", message="worker died", retryable=True
+                        ),
+                    )
+                ).encode()
+            )
+        elif request.path == "/v1/jobs/cancel":
+            if payload["job_id"] != "job-1":
+                request.send_response(404)
+                request.end_headers()
+                return
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            request.wfile.write(b'{"job_id": "job-1"}')
+        elif request.path == "/v1/jobs/query_events":
+            assert payload["job_id"] == "job-1"
+            request.send_response(200)
+            request.send_header("Content-Type", "application/vnd.apache.arrow.stream")
+            request.end_headers()
+            request.wfile.write(events_body)
+        else:
+            request.send_response(404)
+            request.end_headers()
+
+    with mock_lancedb_connection(handler) as db:
+        jobs = db.list_jobs()
+        assert [job.job_id for job in jobs] == ["job-1", "job-2"]
+        assert jobs[0].state == "running"
+        assert jobs[0].table == "t1"
+        assert jobs[1].state == "finished"
+
+        description = db.get_job("job-1")
+        assert description.job_type == "create_index"
+        assert description.state == "failed"
+        assert json.loads(description.spec_json) == {"column": "vec"}
+        assert description.failure.message == "worker died"
+        assert description.failure.retryable is True
+        assert db.get_job("missing") is None
+
+        assert db.cancel_job("job-1") is True
+        assert db.cancel_job("missing") is False
+
+        batches = db.job_history("job-1")
+        assert len(batches) == 1
+        assert batches[0].num_rows == 2
+        assert batches[0].column("state").to_pylist() == ["created", "done"]
+
+        job = db.job("job-1")
+        assert job.id == "job-1"
+        assert job.status() == "failed"
+        with pytest.raises(JobFailedError, match="worker died"):
+            job.wait(timeout=timedelta(seconds=5))

python/src/connection.rs

@@ -13,7 +13,11 @@ use crate::{
     runtime::future_into_py,
     table::Table,
 };
-use arrow::{datatypes::Schema, ffi_stream::ArrowArrayStreamReader, pyarrow::FromPyArrow};
+use arrow::{
+    datatypes::Schema,
+    ffi_stream::ArrowArrayStreamReader,
+    pyarrow::{FromPyArrow, ToPyArrow},
+};
 use lancedb::{
     connection::Connection as LanceConnection,
     connection::NamespaceClientPushdownOperation,
@@ -24,7 +28,7 @@ use pyo3::{
     Bound, FromPyObject, Py, PyAny, PyRef, PyResult, Python,
     exceptions::{PyRuntimeError, PyValueError},
     pyclass, pyfunction, pymethods,
-    types::{PyDict, PyDictMethods},
+    types::{PyDict, PyDictMethods, PyList, PyListMethods},
 };
 
 #[pyclass]
@@ -536,6 +540,55 @@ impl Connection {
             })
         })
     }
+
+    pub fn job(&self, job_id: String) -> PyResult<crate::job::Job> {
+        let inner = self.get_inner()?.clone();
+        Ok(crate::job::Job::new(inner.job(job_id).infer_error()?))
+    }
+
+    pub fn list_jobs(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.get_inner()?.clone();
+        future_into_py(self_.py(), async move {
+            let jobs = inner.list_jobs().await.infer_error()?;
+            Ok(jobs
+                .into_iter()
+                .map(crate::job::JobInfo::from)
+                .collect::<Vec<_>>())
+        })
+    }
+
+    pub fn get_job(self_: PyRef<'_, Self>, job_id: String) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.get_inner()?.clone();
+        future_into_py(self_.py(), async move {
+            let description = inner.get_job(&job_id).await.infer_error()?;
+            Ok(description.map(crate::job::JobDescription::from))
+        })
+    }
+
+    pub fn cancel_job(self_: PyRef<'_, Self>, job_id: String) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.get_inner()?.clone();
+        future_into_py(self_.py(), async move {
+            inner.cancel_job(&job_id).await.infer_error()
+        })
+    }
+
+    #[pyo3(signature = (job_id=None))]
+    pub fn job_history(
+        self_: PyRef<'_, Self>,
+        job_id: Option<String>,
+    ) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.get_inner()?.clone();
+        future_into_py(self_.py(), async move {
+            let batches = inner.job_history(job_id.as_deref()).await.infer_error()?;
+            Python::attach(|py| {
+                let list = PyList::empty(py);
+                for batch in batches {
+                    list.append(batch.to_pyarrow(py)?)?;
+                }
+                Ok(list.unbind())
+            })
+        })
+    }
 }
 
 #[pyfunction]

python/src/job.rs

@@ -28,6 +28,14 @@ impl Job {
         self.inner.id().map(str::to_string)
     }
 
+    pub fn status(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner.clone();
+        future_into_py(
+            self_.py(),
+            async move { inner.status().await.infer_error() },
+        )
+    }
+
     pub fn wait(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.inner.clone();
         future_into_py(self_.py(), async move {
@@ -44,3 +52,94 @@ impl Job {
         })
     }
 }
+
+/// A row from `Connection.list_jobs`: one server-side job.
+#[pyclass(get_all, skip_from_py_object)]
+#[derive(Clone)]
+pub struct JobInfo {
+    job_id: String,
+    table: String,
+    job_type: String,
+    state: String,
+    created_at_millis: i64,
+}
+
+#[pymethods]
+impl JobInfo {
+    fn __repr__(&self) -> String {
+        format!(
+            "JobInfo(job_id={:?}, table={:?}, job_type={:?}, state={:?}, created_at_millis={})",
+            self.job_id, self.table, self.job_type, self.state, self.created_at_millis
+        )
+    }
+}
+
+impl From<lancedb::database::JobInfo> for JobInfo {
+    fn from(info: lancedb::database::JobInfo) -> Self {
+        Self {
+            job_id: info.job_id,
+            table: info.table,
+            job_type: info.job_type,
+            state: info.state,
+            created_at_millis: info.created_at_millis,
+        }
+    }
+}
+
+/// The server's account of why a job failed.
+#[pyclass(get_all, skip_from_py_object)]
+#[derive(Clone)]
+pub struct JobFailureInfo {
+    phase: Option<String>,
+    message: Option<String>,
+    retryable: Option<bool>,
+}
+
+#[pymethods]
+impl JobFailureInfo {
+    fn __repr__(&self) -> String {
+        format!(
+            "JobFailureInfo(phase={:?}, message={:?}, retryable={:?})",
+            self.phase, self.message, self.retryable
+        )
+    }
+}
+
+/// A described job from `Connection.get_job`.
+#[pyclass(get_all, skip_from_py_object)]
+#[derive(Clone)]
+pub struct JobDescription {
+    job_id: String,
+    job_type: String,
+    state: String,
+    creation_ms: i64,
+    spec_json: Option<String>,
+    failure: Option<JobFailureInfo>,
+}
+
+#[pymethods]
+impl JobDescription {
+    fn __repr__(&self) -> String {
+        format!(
+            "JobDescription(job_id={:?}, job_type={:?}, state={:?}, creation_ms={})",
+            self.job_id, self.job_type, self.state, self.creation_ms
+        )
+    }
+}
+
+impl From<lancedb::database::JobDescription> for JobDescription {
+    fn from(description: lancedb::database::JobDescription) -> Self {
+        Self {
+            job_id: description.job_id,
+            job_type: description.job_type,
+            state: description.state,
+            creation_ms: description.creation_ms,
+            spec_json: (!description.spec.is_null()).then(|| description.spec.to_string()),
+            failure: description.failure.map(|failure| JobFailureInfo {
+                phase: failure.phase,
+                message: failure.message,
+                retryable: failure.retryable,
+            }),
+        }
+    }
+}

python/src/lib.rs

@@ -46,6 +46,9 @@ pub fn _lancedb(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_class::<Session>()?;
     m.add_class::<Table>()?;
     m.add_class::<crate::job::Job>()?;
+    m.add_class::<crate::job::JobInfo>()?;
+    m.add_class::<crate::job::JobDescription>()?;
+    m.add_class::<crate::job::JobFailureInfo>()?;
     m.add_class::<PyBlobFile>()?;
     m.add_class::<IndexConfig>()?;
     m.add_class::<Query>()?;