feat: expose structured job progress to SDK callers (#4808)

Job callers currently lose the server's structured progress when
describing, listing, or waiting for a job. Expose Rust and Python
progress snapshots and optional wait observers using the existing
backend poller. Python supports callbacks, a caller-owned tqdm-like bar,
or `progress=True`; `status()` still returns a string and the default
wait behavior is unchanged.

Preserve unknown progress fields and phases from newer servers, and
report omitted counters as unknown. Each wait calibrates a caller-owned
bar from its first complete snapshot, then uses a high-water mark within
the same execution epoch; a new epoch or smaller plan resets the bar.
ETA is best-effort and omitted for missing, stale, paused, or
incompatible samples.

Newly submitted index and refresh handles forward observers through
their freshness wrapper while retaining successful-wait read
consistency. The Jobs guide includes draft Python/Rust snapshot and
observer examples for release; TypeScript progress support remains
tracked under ENT-3500.

Validation on the development EC2 host: the initial 11 Python progress
tests, REST fixture caller, and two Rust progress regressions passed.
After fixing freshness-wrapper forwarding, rebuilt the Python extension
and ran two submitted-handle Python tests plus the new Rust refresh and
index regressions (one test each); all passed. Strict Clippy, rustfmt,
and targeted Ruff passed. The final commit only wraps the new guide
section in an MDX draft and clarifies language support. Repository-wide
Ruff reports 48 existing findings in six unchanged docs/web-tests files.

This publishes the SDK portion of ENT-3500. Server-side live-split and
terminal-progress changes and the combined server/SDK end-to-end
scenario remain in progress. Refs ENT-3500.

BREAKING CHANGE: Rust struct literals for `JobInfo` must add `progress:
None`, `phase: None`, and `paused: false`. Literals for `JobDescription`
must add `progress: None`, `phase: None`, `queue_reason: None`,
`waiting_for_job_id: None`, and `paused_at_millis: None` when that
metadata is unavailable. Existing Python calls remain compatible.

python/src/job.rs

@@ -10,7 +10,7 @@ use arrow::{
 };
 use lancedb::job::JobEventsRequest;
 use pyo3::{
-    Bound, PyAny, PyRef, PyResult, Python,
+    Bound, Py, PyAny, PyRef, PyResult, Python,
     exceptions::PyValueError,
     pyclass, pymethods,
     types::{PyAnyMethods, PyDict, PyDictMethods},
@@ -94,10 +94,30 @@ impl Job {
         )
     }
 
-    pub fn wait(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+    #[pyo3(signature = (progress=None))]
+    pub fn wait(self_: PyRef<'_, Self>, progress: Option<Py<PyAny>>) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.inner.clone();
         future_into_py(self_.py(), async move {
-            let result = inner.wait().await.infer_error()?;
+            let result = if let Some(progress) = progress {
+                let observer = move |description: &lancedb::database::JobDescription| {
+                    Python::attach(|py| {
+                        let snapshot = description.progress_snapshot();
+                        let callback_result: PyResult<()> = (|| {
+                            let raw = snapshot.to_string();
+                            let parsed = parse_json_payload(py, Some(&raw))?
+                                .ok_or_else(|| PyValueError::new_err("missing progress"))?;
+                            progress.bind(py).call1((parsed,))?;
+                            Ok(())
+                        })();
+                        if let Err(error) = callback_result {
+                            log::warn!("job progress callback error: {error}");
+                        }
+                    });
+                };
+                inner.wait_with_progress(observer).await.infer_error()?
+            } else {
+                inner.wait().await.infer_error()?
+            };
             result
                 .map_err(|message| lancedb::Error::Runtime { message })
                 .infer_error()
@@ -164,6 +184,8 @@ pub struct JobInfo {
     job_type: String,
     state: String,
     created_at_millis: i64,
+    _progress_json: Option<String>,
+    paused: bool,
 }
 
 #[pymethods]
@@ -174,16 +196,32 @@ impl JobInfo {
             self.job_id, self.table, self.job_type, self.state, self.created_at_millis
         )
     }
+
+    /// Structured progress for job types that report it. Missing fields are
+    /// unknown, not zero.
+    #[getter]
+    fn progress<'py>(&self, py: Python<'py>) -> PyResult<Option<Bound<'py, PyAny>>> {
+        parse_json_payload(py, self._progress_json.as_deref())
+    }
+
+    /// Whether an operator has parked this job.
+    #[getter]
+    fn paused(&self) -> bool {
+        self.paused
+    }
 }
 
 impl From<lancedb::database::JobInfo> for JobInfo {
     fn from(info: lancedb::database::JobInfo) -> Self {
+        let progress = info.progress_snapshot().to_string();
         Self {
             job_id: info.job_id,
             table: info.table,
             job_type: info.job_type,
             state: info.state,
             created_at_millis: info.created_at_millis,
+            _progress_json: Some(progress),
+            paused: info.paused,
         }
     }
 }
@@ -229,6 +267,7 @@ pub struct JobDescription {
     _spec_json: Option<String>,
     /// Internal: the wire form behind the `result` property.
     _result_json: Option<String>,
+    _progress_json: Option<String>,
     failure: Option<JobFailureInfo>,
 }
 
@@ -246,6 +285,13 @@ impl JobDescription {
         parse_json_payload(py, self._result_json.as_deref())
     }
 
+    /// Structured progress snapshot. Lifecycle and queue fields are present
+    /// even when a job type or older server supplies no counters.
+    #[getter]
+    fn progress<'py>(&self, py: Python<'py>) -> PyResult<Option<Bound<'py, PyAny>>> {
+        parse_json_payload(py, self._progress_json.as_deref())
+    }
+
     fn __repr__(&self, py: Python<'_>) -> PyResult<String> {
         let mut fields = vec![
             format!("job_id={:?}", self.job_id),
@@ -273,6 +319,7 @@ impl JobDescription {
 
 impl From<lancedb::database::JobDescription> for JobDescription {
     fn from(description: lancedb::database::JobDescription) -> Self {
+        let progress_json = description.progress_snapshot().to_string();
         Self {
             job_id: description.job_id,
             job_type: description.job_type,
@@ -283,6 +330,7 @@ impl From<lancedb::database::JobDescription> for JobDescription {
                 .result
                 .filter(|result| !result.is_null())
                 .map(|result| result.to_string()),
+            _progress_json: Some(progress_json),
             failure: description.failure.map(|failure| JobFailureInfo {
                 phase: failure.phase,
                 message: failure.message,