feat: connection-level job operations (#3755)

Adds job operations to the connection surface, building on the Job
handle from #3742: job(id), list_jobs, get_job, cancel_job, and
job_history, plus a non-blocking Job.status(). Implemented on the
Database trait (defaulting to NotSupported), the remote backend
(/v1/jobs), and the Python and Node bindings; job_history returns Arrow
batches.

errors() and progress() are not included.

Tested with mocked endpoints in all three languages.

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

nodejs/__test__/remote.test.ts

@@ -877,3 +877,96 @@ describe("remote connection", () => {
     });
   });
 });
+
+describe("remote connection jobs surface", () => {
+  it("lists, describes, cancels, and reads history", async () => {
+    const { tableFromArrays, tableToIPC } = await import("apache-arrow");
+    const eventsTable = tableFromArrays({ state: ["created", "succeeded"] });
+    const eventsBody = Buffer.from(tableToIPC(eventsTable, "stream"));
+
+    await withMockDatabase(
+      (req, res) => {
+        let body = "";
+        req.on("data", (chunk) => {
+          body += chunk;
+        });
+        req.on("end", () => {
+          const payload = body.length > 0 ? JSON.parse(body) : {};
+          if (req.url === "/v1/jobs/list") {
+            if (payload["page_token"] === undefined) {
+              res
+                .writeHead(200, { "Content-Type": "application/json" })
+                .end(
+                  '{"jobs": [{"job_id": "job-1", "table": "t1", ' +
+                    '"job_type": "create_index", "state": "in_progress", ' +
+                    '"created_at_millis": 1000}], "page_token": "next"}',
+                );
+            } else {
+              res
+                .writeHead(200, { "Content-Type": "application/json" })
+                .end(
+                  '{"jobs": [{"job_id": "job-2", "table": "t2", ' +
+                    '"job_type": "create_index", "state": "succeeded", ' +
+                    '"created_at_millis": 2000}]}',
+                );
+            }
+          } else if (req.url === "/v1/jobs/describe") {
+            if (payload["job_id"] !== "job-1") {
+              res.writeHead(404).end("no such job");
+              return;
+            }
+            res
+              .writeHead(200, { "Content-Type": "application/json" })
+              .end(
+                '{"job_id": "job-1", "job_type": "create_index", ' +
+                  '"job_state": "FAILED", "creation_ms": 1000, ' +
+                  '"spec": {"column": "vec"}, "failure": {"phase": "execute", ' +
+                  '"message": "worker died", "retryable": true}}',
+              );
+          } else if (req.url === "/v1/jobs/cancel") {
+            if (payload["job_id"] !== "job-1") {
+              res.writeHead(404).end("no such job");
+              return;
+            }
+            res
+              .writeHead(200, { "Content-Type": "application/json" })
+              .end('{"job_id": "job-1"}');
+          } else if (req.url === "/v1/jobs/query_events") {
+            res
+              .writeHead(200, {
+                "Content-Type": "application/vnd.apache.arrow.stream",
+              })
+              .end(eventsBody);
+          } else {
+            res.writeHead(404).end();
+          }
+        });
+      },
+      async (db) => {
+        const jobs = await db.listJobs();
+        expect(jobs.map((job) => job.jobId)).toEqual(["job-1", "job-2"]);
+        expect(jobs[0].state).toEqual("running");
+        expect(jobs[1].state).toEqual("finished");
+
+        const description = await db.getJob("job-1");
+        expect(description?.state).toEqual("failed");
+        expect(JSON.parse(description?.specJson ?? "")).toEqual({
+          column: "vec",
+        });
+        expect(description?.failure?.message).toEqual("worker died");
+        expect(await db.getJob("missing")).toBeNull();
+
+        expect(await db.cancelJob("job-1")).toBe(true);
+        expect(await db.cancelJob("missing")).toBe(false);
+
+        const history = await db.jobHistory("job-1");
+        expect(history.numRows).toEqual(2);
+
+        const job = db.job("job-1");
+        expect(job.id).toEqual("job-1");
+        expect(await job.status()).toEqual("failed");
+        await expect(job.wait()).rejects.toThrow("worker died");
+      },
+    );
+  });
+});

nodejs/lancedb/connection.ts

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
+import { tableFromIPC } from "apache-arrow";
 import {
   Data,
   SchemaLike,
@@ -20,6 +21,9 @@ import type {
   CreateNamespaceResponse,
   DescribeNamespaceResponse,
   DropNamespaceResponse,
+  Job,
+  JobDescription,
+  JobInfo,
   ListNamespacesResponse,
 } from "./native";
 export type {
@@ -436,6 +440,40 @@ export abstract class Connection {
     newName: string,
     options?: RenameTableOptions,
   ): Promise<void>;
+
+  /**
+   * A {@link Job} handle for a server-side job by id.
+   *
+   * The handle is constructed without a server round trip; an unknown id
+   * surfaces when the handle is used. Dropping the handle has no effect on
+   * the job itself.
+   */
+  abstract job(jobId: string): Job;
+
+  /** List server-side jobs across the database's tables. */
+  abstract listJobs(): Promise<JobInfo[]>;
+
+  /**
+   * Describe a single server-side job by id.
+   *
+   * Resolves to `null` when the server has no such job.
+   */
+  abstract getJob(jobId: string): Promise<JobDescription | null>;
+
+  /**
+   * Request cancellation of a server-side job by id.
+   *
+   * Resolves to true if the server accepted the cancellation, false if no
+   * such job exists. Cancelling an already-terminal job is a no-op success.
+   */
+  abstract cancelJob(jobId: string): Promise<boolean>;
+
+  /**
+   * The lifecycle event history of a server-side job, as an Arrow table.
+   *
+   * Lists history across all jobs when `jobId` is omitted.
+   */
+  abstract jobHistory(jobId?: string): Promise<ArrowTable>;
 }
 
 /** @hideconstructor */
@@ -722,6 +760,30 @@ export class LocalConnection extends Connection {
       options?.newNamespacePath,
     );
   }
+
+  job(jobId: string): Job {
+    return this.inner.job(jobId);
+  }
+
+  async listJobs(): Promise<JobInfo[]> {
+    return this.inner.listJobs();
+  }
+
+  async getJob(jobId: string): Promise<JobDescription | null> {
+    return this.inner.getJob(jobId);
+  }
+
+  async cancelJob(jobId: string): Promise<boolean> {
+    return this.inner.cancelJob(jobId);
+  }
+
+  async jobHistory(jobId?: string): Promise<ArrowTable> {
+    const buf = await this.inner.jobHistory(jobId);
+    if (buf.length === 0) {
+      return new ArrowTable();
+    }
+    return tableFromIPC(buf);
+  }
 }
 
 /**

nodejs/lancedb/index.ts

@@ -85,7 +85,13 @@ export {
   RenameTableOptions,
 } from "./connection";
 
-export { Job, Session } from "./native.js";
+export {
+  Job,
+  JobDescription,
+  JobFailureInfo,
+  JobInfo,
+  Session,
+} from "./native.js";
 
 export {
   ExecutableQuery,

nodejs/src/connection.rs

@@ -340,6 +340,69 @@ impl Connection {
         self.get_inner()?.drop_all_tables(&ns).await.default_error()
     }
 
+    /// A `Job` handle for a server-side job by id.
+    ///
+    /// The handle is constructed without a server round trip; an unknown id
+    /// surfaces when the handle is used.
+    #[napi]
+    pub fn job(&self, job_id: String) -> napi::Result<crate::job::Job> {
+        let job = self.get_inner()?.job(job_id).default_error()?;
+        Ok(crate::job::Job::new(job))
+    }
+
+    /// List server-side jobs across the database's tables.
+    #[napi(catch_unwind)]
+    pub async fn list_jobs(&self) -> napi::Result<Vec<crate::job::JobInfo>> {
+        let jobs = self.get_inner()?.list_jobs().await.default_error()?;
+        Ok(jobs.into_iter().map(Into::into).collect())
+    }
+
+    /// Describe a single server-side job by id. `null` when the server has
+    /// no such job.
+    #[napi(catch_unwind)]
+    pub async fn get_job(
+        &self,
+        job_id: String,
+    ) -> napi::Result<Option<crate::job::JobDescription>> {
+        let description = self.get_inner()?.get_job(&job_id).await.default_error()?;
+        Ok(description.map(Into::into))
+    }
+
+    /// Request cancellation of a server-side job by id. Returns true if the
+    /// server accepted the cancellation, false if no such job exists.
+    #[napi(catch_unwind)]
+    pub async fn cancel_job(&self, job_id: String) -> napi::Result<bool> {
+        self.get_inner()?.cancel_job(&job_id).await.default_error()
+    }
+
+    /// The lifecycle event history of a server-side job (all jobs when
+    /// `job_id` is null), as an Arrow IPC stream buffer. Empty when there is
+    /// no history.
+    #[napi(catch_unwind)]
+    pub async fn job_history(&self, job_id: Option<String>) -> napi::Result<Buffer> {
+        let batches = self
+            .get_inner()?
+            .job_history(job_id.as_deref())
+            .await
+            .default_error()?;
+        let Some(first) = batches.first() else {
+            return Ok(Buffer::from(Vec::<u8>::new()));
+        };
+        let mut out = Vec::new();
+        let mut writer = arrow_ipc::writer::StreamWriter::try_new(&mut out, &first.schema())
+            .map_err(|e| napi::Error::from_reason(e.to_string()))?;
+        for batch in &batches {
+            writer
+                .write(batch)
+                .map_err(|e| napi::Error::from_reason(e.to_string()))?;
+        }
+        writer
+            .finish()
+            .map_err(|e| napi::Error::from_reason(e.to_string()))?;
+        drop(writer);
+        Ok(Buffer::from(out))
+    }
+
     #[napi(catch_unwind)]
     /// Describe a namespace and return its properties.
     pub async fn describe_namespace(

nodejs/src/job.rs

@@ -30,6 +30,17 @@ impl Job {
         self.inner.id().map(str::to_string)
     }
 
+    /// The operation's current lifecycle state: "running", "finished",
+    /// "failed", or "cancelled".
+    ///
+    /// A point snapshot; unlike {@link Job.wait} it does not block or reject
+    /// on a terminal failure state. States a newer server reports that this
+    /// client version does not know pass through as-is.
+    #[napi(catch_unwind)]
+    pub async fn status(&self) -> napi::Result<String> {
+        self.inner.status().await.default_error()
+    }
+
     /// Wait until the operation reaches a terminal state.
     #[napi(catch_unwind)]
     pub async fn wait(&self) -> napi::Result<()> {
@@ -42,3 +53,71 @@ impl Job {
         self.inner.cancel().await.default_error()
     }
 }
+
+/// A row from `Connection.listJobs`: one server-side job.
+#[napi(object)]
+pub struct JobInfo {
+    /// The job id -- what `Connection.getJob` and `Connection.cancelJob`
+    /// accept.
+    pub job_id: String,
+    /// The table the job runs against, without URI or namespace.
+    pub table: String,
+    pub job_type: String,
+    /// Lifecycle state: "running", "finished", "failed", or "cancelled".
+    pub state: String,
+    /// When the job was created, in milliseconds since the epoch.
+    pub created_at_millis: i64,
+}
+
+impl From<lancedb::database::JobInfo> for JobInfo {
+    fn from(info: lancedb::database::JobInfo) -> Self {
+        Self {
+            job_id: info.job_id,
+            table: info.table,
+            job_type: info.job_type,
+            state: info.state,
+            created_at_millis: info.created_at_millis,
+        }
+    }
+}
+
+/// The server's account of why a job failed.
+#[napi(object)]
+pub struct JobFailureInfo {
+    pub phase: Option<String>,
+    pub message: Option<String>,
+    pub retryable: Option<bool>,
+}
+
+/// A described job from `Connection.getJob`.
+#[napi(object)]
+pub struct JobDescription {
+    pub job_id: String,
+    pub job_type: String,
+    /// Lifecycle state: "running", "finished", "failed", or "cancelled".
+    pub state: String,
+    /// When the job was created, in milliseconds since the epoch.
+    pub creation_ms: i64,
+    /// The job-type-specific specification as a JSON string, when present.
+    pub spec_json: Option<String>,
+    /// Why the job failed, when the job is failed and the server reports a
+    /// reason.
+    pub failure: Option<JobFailureInfo>,
+}
+
+impl From<lancedb::database::JobDescription> for JobDescription {
+    fn from(description: lancedb::database::JobDescription) -> Self {
+        Self {
+            job_id: description.job_id,
+            job_type: description.job_type,
+            state: description.state,
+            creation_ms: description.creation_ms,
+            spec_json: (!description.spec.is_null()).then(|| description.spec.to_string()),
+            failure: description.failure.map(|failure| JobFailureInfo {
+                phase: failure.phase,
+                message: failure.message,
+                retryable: failure.retryable,
+            }),
+        }
+    }
+}