feat(sql): expose durable job ids on query handles (#4941)

Expose the durable SQL registry id separately from the connection-scoped
query UUID. Rust and Python query handles and descriptions expose
optional `job_id`; result readers preserve it in schema metadata under
`x-lancedb-sql-job-id`, without adding business columns. Clients can
reopen that id with `open_job`.

Poll success and failure metadata update the same handle. A failure
before the first successful poll retains its job id and original
structured error. The cancellation documentation distinguishes the
Flight waiter from `cancel_job(job_id)`.

Validation on the designated EC2: the two focused error interoperability
tests and Ruff passed on `a8798e81`; delivery has the identical tree.
The preceding 51 Rust SQL tests and earlier 31 native Python SQL tests
are reused for unchanged coverage. Matching-runtime independent
acceptance remains pending.

Companion server change: https://github.com/lancedb/sophon/pull/10049.

Refs ENT-3562.

python/python/lancedb/_lancedb.pyi

@@ -501,6 +501,8 @@ class JobDescription:
 class SqlQuery:
     @property
     def id(self) -> UUID: ...
+    @property
+    def job_id(self) -> Optional[str]: ...
     async def describe(self) -> QueryDescription: ...
     async def reader(self) -> RecordBatchStream: ...
     async def cancel(self) -> None: ...
@@ -509,6 +511,8 @@ class QueryDescription:
     @property
     def id(self) -> UUID: ...
     @property
+    def job_id(self) -> Optional[str]: ...
+    @property
     def status(self) -> str: ...
     @property
     def progress(self) -> Optional[float]: ...

python/python/lancedb/db.py

@@ -1515,7 +1515,10 @@ class DBConnection(EnforceOverrides):
 
         This submits through :meth:`execute_query_async` and waits until the
         initial result stream is readable. It does not wait for the full query
-        to finish.
+        to finish. A statement that ran as a server-side SQL job exposes the
+        durable Job Registry id in the reader schema metadata under
+        ``x-lancedb-sql-job-id``. Use :meth:`execute_query_async` when the
+        query handle itself is needed.
         """
         return self.execute_query_async(
             query,
@@ -1554,6 +1557,9 @@ class DBConnection(EnforceOverrides):
         stops a directly streamed query; deferred execution requires explicit
         cancellation. Open ``reader()`` directly: a streamed query needs its
         reader to make progress and reports ``running`` until the stream ends.
+        Once a server-side SQL job exists, ``describe().job_id`` exposes its
+        durable Job Registry id while the query is still running. This is
+        separate from the connection-scoped query ``id``.
 
         LanceDB Embedded connections do not support SQL.
         """
@@ -4128,7 +4134,10 @@ class AsyncConnection(object):
 
         This submits through :meth:`execute_query_async` and waits until the
         initial result stream is readable. It does not wait for the full query
-        to finish.
+        to finish. A statement that ran as a server-side SQL job exposes the
+        durable Job Registry id in the reader schema metadata under
+        ``x-lancedb-sql-job-id``. Use :meth:`execute_query_async` when the
+        query handle itself is needed.
         """
         submitted = await self.execute_query_async(
             query,

python/python/lancedb/sql.py

@@ -104,6 +104,15 @@ class AsyncQuery:
         """The stable identifier scoped to the connection that submitted it."""
         return self._inner.id
 
+    @property
+    def job_id(self) -> Optional[str]:
+        """The durable Job Registry id once the server reports this query as a job.
+
+        A running query reports it after :meth:`describe` observes the latest
+        server status. This is separate from the connection-scoped ``id``.
+        """
+        return self._inner.job_id
+
     async def describe(self) -> QueryDescription:
         """Get a point-in-time description of the query."""
         return await self._inner.describe()
@@ -118,7 +127,12 @@ class AsyncQuery:
         return AsyncRecordBatchReader(await self._inner.reader())
 
     async def cancel(self) -> None:
-        """Request cancellation of the query."""
+        """Request cancellation of this query's Flight execution.
+
+        A durable server-side job may reject this operation because its Flight
+        waiter cannot cancel the registry job. Use ``connection.cancel_job()``
+        with ``job_id`` to cancel that job.
+        """
         await self._inner.cancel()
 
 
@@ -133,6 +147,15 @@ class Query:
         """The stable identifier scoped to the connection that submitted it."""
         return self._inner.id
 
+    @property
+    def job_id(self) -> Optional[str]:
+        """The durable Job Registry id once the server reports this query as a job.
+
+        A running query reports it after :meth:`describe` observes the latest
+        server status. This is separate from the connection-scoped ``id``.
+        """
+        return self._inner.job_id
+
     def describe(self) -> QueryDescription:
         """Get a point-in-time description of the query."""
         return LOOP.run(self._inner.describe())
@@ -159,7 +182,12 @@ class Query:
         return pa.RecordBatchReader.from_batches(reader.schema, batches())
 
     def cancel(self) -> None:
-        """Request cancellation of the query."""
+        """Request cancellation of this query's Flight execution.
+
+        A durable server-side job may reject this operation because its Flight
+        waiter cannot cancel the registry job. Use ``connection.cancel_job()``
+        with ``job_id`` to cancel that job.
+        """
         LOOP.run(self._inner.cancel())
 
 

python/python/tests/test_sql.py

@@ -20,6 +20,7 @@ NIL_QUERY_ID = UUID(int=0)
 
 class FakeNativeQuery:
     id = UUID("0198f1b2-c3d4-7e5f-8123-456789abcdef")
+    job_id = None
 
     async def reader(self):
         return pa.table({"value": [1, 2]})
@@ -68,6 +69,15 @@ def test_query_id_is_uuid():
     assert Query(query).id == query.id
 
 
+def test_query_job_id_is_optional_and_separate_from_query_id():
+    native = FakeNativeQuery()
+    native.job_id = "j1-durable"
+    query = AsyncQuery(native)
+    assert query.id != query.job_id
+    assert query.job_id == "j1-durable"
+    assert Query(query).job_id == "j1-durable"
+
+
 def test_connection_serializes_sql_host_override():
     endpoint = "grpc+tls://sql.example.com:10026"
     restored = lancedb.deserialize_conn(

python/src/sql.rs

@@ -31,6 +31,11 @@ impl Query {
         self.inner.id()
     }
 
+    #[getter]
+    pub fn job_id(&self) -> Option<String> {
+        self.inner.job_id()
+    }
+
     pub fn describe(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.inner.clone();
         future_into_py(self_.py(), async move {
@@ -63,6 +68,7 @@ impl Query {
 #[derive(Clone)]
 pub struct QueryDescription {
     id: Uuid,
+    job_id: Option<String>,
     status: String,
     progress: Option<f64>,
     expires_at: Option<DateTime<Utc>>,
@@ -72,8 +78,8 @@ pub struct QueryDescription {
 impl QueryDescription {
     fn __repr__(&self) -> String {
         format!(
-            "QueryDescription(id={:?}, status={:?}, progress={:?}, expires_at={:?})",
-            self.id, self.status, self.progress, self.expires_at
+            "QueryDescription(id={:?}, job_id={:?}, status={:?}, progress={:?}, expires_at={:?})",
+            self.id, self.job_id, self.status, self.progress, self.expires_at
         )
     }
 }
@@ -82,6 +88,7 @@ impl From<lancedb::sql::QueryDescription> for QueryDescription {
     fn from(description: lancedb::sql::QueryDescription) -> Self {
         Self {
             id: description.id,
+            job_id: description.job_id,
             status: description.status.to_string(),
             progress: description.progress,
             expires_at: description.expires_at,