From edc49e60e2e14a53ff67ff62d33b68981d8a6c6f Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Thu, 5 Nov 2026 15:16:28 +0800 Subject: [PATCH] feat: expose Function failure tracebacks (#4861) Function refresh failures and skipped-row diagnostics lose the traceback when decoded by the SDK. Expose the server-provided optional `traceback` on job failure details and Function error records in Rust, Python, and Node.js, while keeping existing compact error messages. Responses from older servers remain supported when the field is absent. Tracebacks may be truncated by the server. The Rust `Error::JobFailed` variant now stores `Box` so the added diagnostics do not enlarge every `Result` error payload. --- docs/src/js/interfaces/FunctionErrorRecord.md | 10 ++++++++++ docs/src/js/interfaces/JobFailureInfo.md | 10 ++++++++++ nodejs/src/job.rs | 3 +++ nodejs/src/table.rs | 3 +++ python/python/lancedb/_lancedb.pyi | 6 ++++++ python/python/tests/test_remote_db.py | 13 +++++++++++++ python/src/job.rs | 3 +++ python/src/table.rs | 3 +++ rust/lancedb/src/error.rs | 4 +++- rust/lancedb/src/function.rs | 3 +++ rust/lancedb/src/job.rs | 2 +- rust/lancedb/src/remote/job.rs | 13 +++++++++---- rust/lancedb/src/remote/sql.rs | 5 +++-- rust/lancedb/src/remote/table.rs | 14 ++++++++++++-- 14 files changed, 82 insertions(+), 10 deletions(-) diff --git a/docs/src/js/interfaces/FunctionErrorRecord.md b/docs/src/js/interfaces/FunctionErrorRecord.md index 3a26534a0..16198e70e 100644 --- a/docs/src/js/interfaces/FunctionErrorRecord.md +++ b/docs/src/js/interfaces/FunctionErrorRecord.md @@ -90,3 +90,13 @@ detail was capped. ```ts tableVersion: number; ``` + +*** + +### traceback? + +```ts +optional traceback: string; +``` + +User-code traceback, possibly truncated by the server. diff --git a/docs/src/js/interfaces/JobFailureInfo.md b/docs/src/js/interfaces/JobFailureInfo.md index 5683ee5f0..fe9ce04e3 100644 --- a/docs/src/js/interfaces/JobFailureInfo.md +++ b/docs/src/js/interfaces/JobFailureInfo.md @@ -31,3 +31,13 @@ optional phase: string; ```ts optional retryable: boolean; ``` + +*** + +### traceback? + +```ts +optional traceback: string; +``` + +User-code traceback, possibly truncated by the server. diff --git a/nodejs/src/job.rs b/nodejs/src/job.rs index b8b2d9e94..485035d33 100644 --- a/nodejs/src/job.rs +++ b/nodejs/src/job.rs @@ -141,6 +141,7 @@ impl Job { self.inner.failure().map(|failure| JobFailureInfo { phase: failure.phase, message: failure.message, + traceback: failure.traceback, retryable: failure.retryable, }) } @@ -218,5 +219,7 @@ impl From for JobInfo { pub struct JobFailureInfo { pub phase: Option, pub message: Option, + /// User-code traceback, possibly truncated by the server. + pub traceback: Option, pub retryable: Option, } diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs index 9c9fd5156..c480ba042 100644 --- a/nodejs/src/table.rs +++ b/nodejs/src/table.rs @@ -1530,6 +1530,8 @@ pub struct FunctionErrorRecord { pub table_version: i64, pub error_type: String, pub error_message: String, + /// User-code traceback, possibly truncated by the server. + pub traceback: Option, pub created_at_millis: i64, } @@ -1545,6 +1547,7 @@ impl From for FunctionErrorRecord { table_version: record.table_version as i64, error_type: record.error_type, error_message: record.error_message, + traceback: record.traceback, created_at_millis: record.created_at_millis, } } diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 351933ed5..ed62b6b1d 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -465,6 +465,10 @@ class JobFailureInfo: @property def message(self) -> Optional[str]: ... @property + def traceback(self) -> Optional[str]: + """User-code traceback, possibly truncated by the server.""" + ... + @property def retryable(self) -> Optional[bool]: ... @property def sql_job_id(self) -> Optional[str]: ... @@ -1014,6 +1018,8 @@ class FunctionErrorRecord: table_version: int error_type: str error_message: str + traceback: Optional[str] + """User-code traceback, possibly truncated by the server.""" created_at_millis: int class FunctionErrorFragment: diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 8aec4d185..8178aec62 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -2055,6 +2055,9 @@ def test_remote_function_errors_lists_the_rows_a_refresh_skipped(): "table_version": 11, "error_type": "ValueError", "error_message": "bad input 'x'", + "traceback": ( + "Traceback (most recent call last):\nValueError: bad input 'x'" + ), "created_at_millis": 1700000000000, } ], @@ -2102,6 +2105,9 @@ def test_remote_function_errors_lists_the_rows_a_refresh_skipped(): assert errors.records[0].row_id == 42 assert errors.records[0].row_address == 12884901897 assert errors.records[0].function_version == "2" + assert errors.records[0].traceback == ( + "Traceback (most recent call last):\nValueError: bad input 'x'" + ) assert repr(errors.records[0]) == ( 'FunctionErrorRecord(job_id="j-7", fragment_id=3, row_offset=9, ' 'row_address=12884901897, column="embedding", ' @@ -5128,6 +5134,10 @@ def test_remote_connection_jobs_surface(): failure=dict( phase="execute", message="worker died", + traceback=( + "Traceback (most recent call last):\n" + "RuntimeError: worker died" + ), retryable=True, sql_job_id="sql-job-1", ), @@ -5217,6 +5227,9 @@ def test_remote_connection_jobs_surface(): assert progress["waiting_for_job_id"] == "job-maintenance" assert progress["paused_at_millis"] == 1200 assert job.failure.message == "worker died" + assert job.failure.traceback == ( + "Traceback (most recent call last):\nRuntimeError: worker died" + ) assert job.failure.sql_job_id == "sql-job-1" assert repr(job.failure) == ( 'JobFailureInfo(phase="execute", message="worker died", retryable=True, ' diff --git a/python/src/job.rs b/python/src/job.rs index aa4d8890f..79a9cd524 100644 --- a/python/src/job.rs +++ b/python/src/job.rs @@ -232,6 +232,8 @@ impl From for JobInfo { pub struct JobFailureInfo { phase: Option, message: Option, + /// User-code traceback, possibly truncated by the server. + traceback: Option, retryable: Option, sql_job_id: Option, } @@ -342,6 +344,7 @@ impl From for JobDescription { failure: description.failure.map(|failure| JobFailureInfo { phase: failure.phase, message: failure.message, + traceback: failure.traceback, retryable: failure.retryable, sql_job_id: failure.sql_job_id, }), diff --git a/python/src/table.rs b/python/src/table.rs index 4ccc76f89..44c8017c4 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -622,6 +622,8 @@ pub struct FunctionErrorRecord { pub table_version: u64, pub error_type: String, pub error_message: String, + /// User-code traceback, possibly truncated by the server. + pub traceback: Option, pub created_at_millis: i64, } @@ -662,6 +664,7 @@ impl From for FunctionErrorRecord { table_version: record.table_version, error_type: record.error_type, error_message: record.error_message, + traceback: record.traceback, created_at_millis: record.created_at_millis, } } diff --git a/rust/lancedb/src/error.rs b/rust/lancedb/src/error.rs index c31167148..9c79fcf95 100644 --- a/rust/lancedb/src/error.rs +++ b/rust/lancedb/src/error.rs @@ -159,6 +159,8 @@ pub struct JobFailure { pub phase: Option, /// A human-readable reason, when known. pub message: Option, + /// User-code traceback, possibly truncated by the server. + pub traceback: Option, /// Whether a retry could clear the failure, when known. pub retryable: Option, /// Isolated SQL job that raised this failure, when the server supplies @@ -255,7 +257,7 @@ pub enum Error { #[snafu(display("Job{} failed{failure}", job_id.as_ref().map(|id| format!(" {id}")).unwrap_or_default()))] JobFailed { job_id: Option, - failure: JobFailure, + failure: Box, }, #[snafu(display("Job{} was cancelled", job_id.as_ref().map(|id| format!(" {id}")).unwrap_or_default()))] JobCancelled { job_id: Option }, diff --git a/rust/lancedb/src/function.rs b/rust/lancedb/src/function.rs index 96eb4de10..5f322a8f4 100644 --- a/rust/lancedb/src/function.rs +++ b/rust/lancedb/src/function.rs @@ -1121,6 +1121,9 @@ pub struct FunctionErrorRecord { pub error_type: String, /// The error's text. pub error_message: String, + /// User-code traceback, possibly truncated by the server. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub traceback: Option, /// When the error was recorded, in milliseconds since the epoch. pub created_at_millis: i64, } diff --git a/rust/lancedb/src/job.rs b/rust/lancedb/src/job.rs index 823834798..8644c6a48 100644 --- a/rust/lancedb/src/job.rs +++ b/rust/lancedb/src/job.rs @@ -583,7 +583,7 @@ impl Outcome { Self::Failed(source) => { let (job_id, failure) = match source.as_ref() { Error::JobFailed { job_id, failure } => (job_id.clone(), failure.clone()), - _ => (None, JobFailure::from_source(source)), + _ => (None, Box::new(JobFailure::from_source(source))), }; Err(Error::JobFailed { job_id, failure }) } diff --git a/rust/lancedb/src/remote/job.rs b/rust/lancedb/src/remote/job.rs index a5baab7b0..0c652a3ae 100644 --- a/rust/lancedb/src/remote/job.rs +++ b/rust/lancedb/src/remote/job.rs @@ -74,6 +74,8 @@ pub(super) struct ReportedFailure { #[serde(default)] message: Option, #[serde(default)] + traceback: Option, + #[serde(default)] retryable: Option, #[serde(default)] sql_job_id: Option, @@ -134,6 +136,7 @@ impl ReportedFailure { JobFailure { phase: self.phase, message: self.message, + traceback: self.traceback, retryable: self.retryable, sql_job_id: self.sql_job_id, source: None, @@ -266,10 +269,12 @@ impl JobHandle for RemoteJob { JobState::Failed => { return Err(Error::JobFailed { job_id: Some(self.job_id.clone()), - failure: description - .failure - .map(ReportedFailure::into_job_failure) - .unwrap_or_default(), + failure: Box::new( + description + .failure + .map(ReportedFailure::into_job_failure) + .unwrap_or_default(), + ), }); } JobState::Cancelled => { diff --git a/rust/lancedb/src/remote/sql.rs b/rust/lancedb/src/remote/sql.rs index 94d8bf063..83dedcb6d 100644 --- a/rust/lancedb/src/remote/sql.rs +++ b/rust/lancedb/src/remote/sql.rs @@ -1666,13 +1666,14 @@ pub(super) fn sql_status_error(request_id: &str, status: &tonic::Status) -> Erro if let Some(sql_job_id) = sql_job_id { return Error::JobFailed { job_id: Some(sql_job_id.clone()), - failure: JobFailure { + failure: Box::new(JobFailure { phase: None, message: Some(status.message().into()), + traceback: None, retryable: None, sql_job_id: Some(sql_job_id), source: Some(Arc::new(error)), - }, + }), }; } error diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index b95227db6..af0946600 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -8592,6 +8592,7 @@ mod tests { // The server said only that it failed, so nothing may be invented. assert!(failure.message.is_none(), "{failure:?}"); assert!(failure.phase.is_none(), "{failure:?}"); + assert!(failure.traceback.is_none(), "{failure:?}"); assert!(failure.retryable.is_none(), "{failure:?}"); assert_eq!(err.to_string(), "Job job-err failed"); } @@ -8616,7 +8617,7 @@ mod tests { "/v1/jobs/describe" => http::Response::builder() .status(200) .body( - r#"{"job_id": "job-err", "job_state": "FAILED", "failure": {"phase": "commit", "message": "preempted", "retryable": true}}"# + r#"{"job_id": "job-err", "job_state": "FAILED", "failure": {"phase": "commit", "message": "preempted", "traceback": "Traceback (most recent call last):\nValueError: preempted", "retryable": true}}"# .to_string(), ) .unwrap(), @@ -8634,6 +8635,10 @@ mod tests { panic!("expected JobFailed, got {err:?}"); }; assert_eq!(failure.message.as_deref(), Some("preempted")); + assert_eq!( + failure.traceback.as_deref(), + Some("Traceback (most recent call last):\nValueError: preempted") + ); assert_eq!(failure.phase.as_deref(), Some("commit")); assert_eq!(failure.retryable, Some(true)); assert_eq!(err.to_string(), "Job job-err failed: preempted (in commit)"); @@ -11029,7 +11034,8 @@ mod tests { "row_id": 42, "row_address": 12884901897, "column": "embedding", "function": "embed", "function_version": "2", "table_version": 11, "error_type": "ValueError", - "error_message": "bad input 'x'", "created_at_millis": 1700000000000}], + "error_message": "bad input 'x'", "traceback": "Traceback (most recent call last):\nValueError: bad input 'x'", + "created_at_millis": 1700000000000}], "fragments": [{"job_id": "j-7", "fragment_id": 4, "rows_skipped": 500, "rows_recorded": 100}], "truncated": true}"#, @@ -11060,6 +11066,9 @@ mod tests { table_version: 11, error_type: "ValueError".into(), error_message: "bad input 'x'".into(), + traceback: Some( + "Traceback (most recent call last):\nValueError: bad input 'x'".into() + ), created_at_millis: 1_700_000_000_000, }] ); @@ -11091,6 +11100,7 @@ mod tests { .unwrap(); assert!(legacy.row_id.is_none()); assert!(legacy.row_address.is_none()); + assert!(legacy.traceback.is_none()); // No filter sends no filter, and an empty listing reads as such. let table = Table::new_with_handler("my_table", |request| {