diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index b38ae4a5b..c5afac308 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -2991,6 +2991,69 @@ def test_remote_job_wait_raises_on_failure(): assert exc_info.value.sql_job_id is None +@pytest.mark.asyncio +@pytest.mark.parametrize("sync", [True, False]) +@pytest.mark.parametrize("with_terminal_server", [True, False]) +async def test_remote_job_wait_preserves_terminal_build(sync, with_terminal_server): + from lancedb.exceptions import JobFailedError + + message = "ValueError: corrupt frame at frame_index=207" + server = { + "release": "1.13.0-beta.0", + "git_sha": "2a7874ecd589", + "build_date": "2026-11-12T22:45:43Z", + } + result = {"terminal_server": server} if with_terminal_server else None + + def handler(request): + assert request.path == "/v1/jobs/describe" + request.send_response(200) + request.send_header("Content-Type", "application/json") + request.send_header( + "x-lancedb-server-build", + json.dumps({"release": "2.0.0", "git_sha": "new-query-node"}), + ) + request.end_headers() + request.wfile.write( + json.dumps( + { + "job_id": "job-provenance", + "job_state": "FAILED", + "failure": { + "phase": "function_refresh_attempt_failed", + "message": message, + "traceback": "Traceback: original user code", + "retryable": False, + }, + "result": result, + } + ).encode() + ) + + if sync: + with mock_lancedb_connection(handler) as db: + job = db.open_job("job-provenance") + with pytest.raises(JobFailedError) as caught: + job.wait() + else: + async with mock_lancedb_connection_async(handler) as db: + job = await db.open_job("job-provenance") + with pytest.raises(JobFailedError) as caught: + await job.wait() + + expected = f"Job job-provenance failed: {message}" + if with_terminal_server: + expected += ( + " [server: 1.13.0-beta.0 (git 2a7874ecd589, built 2026-11-12T22:45:43Z)]" + ) + expected += " (in function_refresh_attempt_failed)" + assert str(caught.value) == expected + assert caught.value.job_id == "job-provenance" + assert job.failure.message == message + assert job.failure.traceback == "Traceback: original user code" + assert job.result == result + + @pytest.mark.parametrize("server_version", ["0.2.0", "0.6.0"]) def test_remote_json_fts_config(server_version): from lancedb.index import FTS diff --git a/rust/lancedb/src/remote/job.rs b/rust/lancedb/src/remote/job.rs index c546ce4d1..7e59834d0 100644 --- a/rust/lancedb/src/remote/job.rs +++ b/rust/lancedb/src/remote/job.rs @@ -322,14 +322,32 @@ impl JobHandle for RemoteJob { match description.state() { JobState::Done => return Ok(description.into_terminal_result(request_id)), JobState::Failed => { + let mut failure = description + .failure + .map(ReportedFailure::into_job_failure) + .unwrap_or_default(); + // The responding query node may have upgraded since this job + // failed. Only its persisted terminal writer identifies the failure. + if let Some(server) = description + .result + .as_ref() + .and_then(|result| result.get("terminal_server")) + .and_then(|server| { + serde_json::from_value::( + server.clone(), + ) + .ok() + }) + { + let provenance = format!("[server: {server}]"); + failure.message = Some(match failure.message { + Some(message) => format!("{message} {provenance}"), + None => provenance, + }); + } return Err(Error::JobFailed { job_id: Some(self.job_id.clone()), - failure: Box::new( - description - .failure - .map(ReportedFailure::into_job_failure) - .unwrap_or_default(), - ), + failure: Box::new(failure), }); } JobState::Cancelled => { @@ -662,4 +680,72 @@ mod tests { assert_eq!(failure.retryable, Some(true)); assert_eq!(failure.sql_job_id.as_deref(), Some("job-1")); } + + #[tokio::test] + async fn wait_failure_uses_persisted_terminal_build() { + let server = serde_json::json!({ + "release": "1.13.0-beta.0", + "git_sha": "2a7874ecd589", + "build_date": "2026-11-12T22:45:43Z" + }); + for result in [ + serde_json::Value::Null, + serde_json::json!({"terminal_server": null}), + serde_json::json!({"terminal_server": {"release": "incomplete"}}), + serde_json::json!({"terminal_server": server}), + ] { + let known_build = result["terminal_server"] == server; + let response = serde_json::json!({ + "job_id": "job-1", + "job_state": "FAILED", + "failure": { + "phase": "function_refresh_attempt_failed", + "message": "ValueError: corrupt frame at frame_index=207", + "traceback": "Traceback: original user code", + "retryable": false + }, + "result": result + }); + let client = client_with_handler(move |request| { + assert_eq!(request.url().path(), "/v1/jobs/describe"); + http::Response::builder() + .status(200) + .header( + "x-lancedb-server-build", + r#"{"release":"2.0.0","git_sha":"new-query-node"}"#, + ) + .body(response.to_string()) + .unwrap() + }); + let job = RemoteJob::new(client, "job-1".to_string()); + let description = job.describe().await.unwrap(); + assert_eq!( + description.result.as_ref(), + (!result.is_null()).then_some(&result) + ); + assert_eq!( + description.failure.unwrap().message.as_deref(), + Some("ValueError: corrupt frame at frame_index=207") + ); + let error = job.wait().await.err().expect("failed job"); + let mut expected = + "Job job-1 failed: ValueError: corrupt frame at frame_index=207".to_string(); + if known_build { + expected.push_str( + " [server: 1.13.0-beta.0 (git 2a7874ecd589, built 2026-11-12T22:45:43Z)]", + ); + } + expected.push_str(" (in function_refresh_attempt_failed)"); + assert_eq!(error.to_string(), expected); + let crate::error::Error::JobFailed { job_id, failure } = error else { + panic!("expected JobFailed"); + }; + assert_eq!(job_id.as_deref(), Some("job-1")); + assert_eq!( + failure.traceback.as_deref(), + Some("Traceback: original user code") + ); + assert_eq!(failure.retryable, Some(false)); + } + } }