diff --git a/docs/src/js/interfaces/FunctionErrorRecord.md b/docs/src/js/interfaces/FunctionErrorRecord.md index 3a26534a0..16198e70e 100644 --- a/docs/src/js/interfaces/FunctionErrorRecord.md +++ b/docs/src/js/interfaces/FunctionErrorRecord.md @@ -90,3 +90,13 @@ detail was capped. ```ts tableVersion: number; ``` + +*** + +### traceback? + +```ts +optional traceback: string; +``` + +User-code traceback, possibly truncated by the server. diff --git a/docs/src/js/interfaces/JobFailureInfo.md b/docs/src/js/interfaces/JobFailureInfo.md index 5683ee5f0..fe9ce04e3 100644 --- a/docs/src/js/interfaces/JobFailureInfo.md +++ b/docs/src/js/interfaces/JobFailureInfo.md @@ -31,3 +31,13 @@ optional phase: string; ```ts optional retryable: boolean; ``` + +*** + +### traceback? + +```ts +optional traceback: string; +``` + +User-code traceback, possibly truncated by the server. diff --git a/nodejs/src/job.rs b/nodejs/src/job.rs index b8b2d9e94..485035d33 100644 --- a/nodejs/src/job.rs +++ b/nodejs/src/job.rs @@ -141,6 +141,7 @@ impl Job { self.inner.failure().map(|failure| JobFailureInfo { phase: failure.phase, message: failure.message, + traceback: failure.traceback, retryable: failure.retryable, }) } @@ -218,5 +219,7 @@ impl From for JobInfo { pub struct JobFailureInfo { pub phase: Option, pub message: Option, + /// User-code traceback, possibly truncated by the server. + pub traceback: Option, pub retryable: Option, } diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs index 9c9fd5156..c480ba042 100644 --- a/nodejs/src/table.rs +++ b/nodejs/src/table.rs @@ -1530,6 +1530,8 @@ pub struct FunctionErrorRecord { pub table_version: i64, pub error_type: String, pub error_message: String, + /// User-code traceback, possibly truncated by the server. + pub traceback: Option, pub created_at_millis: i64, } @@ -1545,6 +1547,7 @@ impl From for FunctionErrorRecord { table_version: record.table_version as i64, error_type: record.error_type, error_message: record.error_message, + traceback: record.traceback, created_at_millis: record.created_at_millis, } } diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 351933ed5..ed62b6b1d 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -465,6 +465,10 @@ class JobFailureInfo: @property def message(self) -> Optional[str]: ... @property + def traceback(self) -> Optional[str]: + """User-code traceback, possibly truncated by the server.""" + ... + @property def retryable(self) -> Optional[bool]: ... @property def sql_job_id(self) -> Optional[str]: ... @@ -1014,6 +1018,8 @@ class FunctionErrorRecord: table_version: int error_type: str error_message: str + traceback: Optional[str] + """User-code traceback, possibly truncated by the server.""" created_at_millis: int class FunctionErrorFragment: diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 8aec4d185..8178aec62 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -2055,6 +2055,9 @@ def test_remote_function_errors_lists_the_rows_a_refresh_skipped(): "table_version": 11, "error_type": "ValueError", "error_message": "bad input 'x'", + "traceback": ( + "Traceback (most recent call last):\nValueError: bad input 'x'" + ), "created_at_millis": 1700000000000, } ], @@ -2102,6 +2105,9 @@ def test_remote_function_errors_lists_the_rows_a_refresh_skipped(): assert errors.records[0].row_id == 42 assert errors.records[0].row_address == 12884901897 assert errors.records[0].function_version == "2" + assert errors.records[0].traceback == ( + "Traceback (most recent call last):\nValueError: bad input 'x'" + ) assert repr(errors.records[0]) == ( 'FunctionErrorRecord(job_id="j-7", fragment_id=3, row_offset=9, ' 'row_address=12884901897, column="embedding", ' @@ -5128,6 +5134,10 @@ def test_remote_connection_jobs_surface(): failure=dict( phase="execute", message="worker died", + traceback=( + "Traceback (most recent call last):\n" + "RuntimeError: worker died" + ), retryable=True, sql_job_id="sql-job-1", ), @@ -5217,6 +5227,9 @@ def test_remote_connection_jobs_surface(): assert progress["waiting_for_job_id"] == "job-maintenance" assert progress["paused_at_millis"] == 1200 assert job.failure.message == "worker died" + assert job.failure.traceback == ( + "Traceback (most recent call last):\nRuntimeError: worker died" + ) assert job.failure.sql_job_id == "sql-job-1" assert repr(job.failure) == ( 'JobFailureInfo(phase="execute", message="worker died", retryable=True, ' diff --git a/python/src/job.rs b/python/src/job.rs index aa4d8890f..79a9cd524 100644 --- a/python/src/job.rs +++ b/python/src/job.rs @@ -232,6 +232,8 @@ impl From for JobInfo { pub struct JobFailureInfo { phase: Option, message: Option, + /// User-code traceback, possibly truncated by the server. + traceback: Option, retryable: Option, sql_job_id: Option, } @@ -342,6 +344,7 @@ impl From for JobDescription { failure: description.failure.map(|failure| JobFailureInfo { phase: failure.phase, message: failure.message, + traceback: failure.traceback, retryable: failure.retryable, sql_job_id: failure.sql_job_id, }), diff --git a/python/src/table.rs b/python/src/table.rs index 4ccc76f89..44c8017c4 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -622,6 +622,8 @@ pub struct FunctionErrorRecord { pub table_version: u64, pub error_type: String, pub error_message: String, + /// User-code traceback, possibly truncated by the server. + pub traceback: Option, pub created_at_millis: i64, } @@ -662,6 +664,7 @@ impl From for FunctionErrorRecord { table_version: record.table_version, error_type: record.error_type, error_message: record.error_message, + traceback: record.traceback, created_at_millis: record.created_at_millis, } } diff --git a/rust/lancedb/src/error.rs b/rust/lancedb/src/error.rs index c31167148..9c79fcf95 100644 --- a/rust/lancedb/src/error.rs +++ b/rust/lancedb/src/error.rs @@ -159,6 +159,8 @@ pub struct JobFailure { pub phase: Option, /// A human-readable reason, when known. pub message: Option, + /// User-code traceback, possibly truncated by the server. + pub traceback: Option, /// Whether a retry could clear the failure, when known. pub retryable: Option, /// Isolated SQL job that raised this failure, when the server supplies @@ -255,7 +257,7 @@ pub enum Error { #[snafu(display("Job{} failed{failure}", job_id.as_ref().map(|id| format!(" {id}")).unwrap_or_default()))] JobFailed { job_id: Option, - failure: JobFailure, + failure: Box, }, #[snafu(display("Job{} was cancelled", job_id.as_ref().map(|id| format!(" {id}")).unwrap_or_default()))] JobCancelled { job_id: Option }, diff --git a/rust/lancedb/src/function.rs b/rust/lancedb/src/function.rs index 96eb4de10..5f322a8f4 100644 --- a/rust/lancedb/src/function.rs +++ b/rust/lancedb/src/function.rs @@ -1121,6 +1121,9 @@ pub struct FunctionErrorRecord { pub error_type: String, /// The error's text. pub error_message: String, + /// User-code traceback, possibly truncated by the server. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub traceback: Option, /// When the error was recorded, in milliseconds since the epoch. pub created_at_millis: i64, } diff --git a/rust/lancedb/src/job.rs b/rust/lancedb/src/job.rs index 823834798..8644c6a48 100644 --- a/rust/lancedb/src/job.rs +++ b/rust/lancedb/src/job.rs @@ -583,7 +583,7 @@ impl Outcome { Self::Failed(source) => { let (job_id, failure) = match source.as_ref() { Error::JobFailed { job_id, failure } => (job_id.clone(), failure.clone()), - _ => (None, JobFailure::from_source(source)), + _ => (None, Box::new(JobFailure::from_source(source))), }; Err(Error::JobFailed { job_id, failure }) } diff --git a/rust/lancedb/src/remote/job.rs b/rust/lancedb/src/remote/job.rs index a5baab7b0..0c652a3ae 100644 --- a/rust/lancedb/src/remote/job.rs +++ b/rust/lancedb/src/remote/job.rs @@ -74,6 +74,8 @@ pub(super) struct ReportedFailure { #[serde(default)] message: Option, #[serde(default)] + traceback: Option, + #[serde(default)] retryable: Option, #[serde(default)] sql_job_id: Option, @@ -134,6 +136,7 @@ impl ReportedFailure { JobFailure { phase: self.phase, message: self.message, + traceback: self.traceback, retryable: self.retryable, sql_job_id: self.sql_job_id, source: None, @@ -266,10 +269,12 @@ impl JobHandle for RemoteJob { JobState::Failed => { return Err(Error::JobFailed { job_id: Some(self.job_id.clone()), - failure: description - .failure - .map(ReportedFailure::into_job_failure) - .unwrap_or_default(), + failure: Box::new( + description + .failure + .map(ReportedFailure::into_job_failure) + .unwrap_or_default(), + ), }); } JobState::Cancelled => { diff --git a/rust/lancedb/src/remote/sql.rs b/rust/lancedb/src/remote/sql.rs index 94d8bf063..83dedcb6d 100644 --- a/rust/lancedb/src/remote/sql.rs +++ b/rust/lancedb/src/remote/sql.rs @@ -1666,13 +1666,14 @@ pub(super) fn sql_status_error(request_id: &str, status: &tonic::Status) -> Erro if let Some(sql_job_id) = sql_job_id { return Error::JobFailed { job_id: Some(sql_job_id.clone()), - failure: JobFailure { + failure: Box::new(JobFailure { phase: None, message: Some(status.message().into()), + traceback: None, retryable: None, sql_job_id: Some(sql_job_id), source: Some(Arc::new(error)), - }, + }), }; } error diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index b95227db6..af0946600 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -8592,6 +8592,7 @@ mod tests { // The server said only that it failed, so nothing may be invented. assert!(failure.message.is_none(), "{failure:?}"); assert!(failure.phase.is_none(), "{failure:?}"); + assert!(failure.traceback.is_none(), "{failure:?}"); assert!(failure.retryable.is_none(), "{failure:?}"); assert_eq!(err.to_string(), "Job job-err failed"); } @@ -8616,7 +8617,7 @@ mod tests { "/v1/jobs/describe" => http::Response::builder() .status(200) .body( - r#"{"job_id": "job-err", "job_state": "FAILED", "failure": {"phase": "commit", "message": "preempted", "retryable": true}}"# + r#"{"job_id": "job-err", "job_state": "FAILED", "failure": {"phase": "commit", "message": "preempted", "traceback": "Traceback (most recent call last):\nValueError: preempted", "retryable": true}}"# .to_string(), ) .unwrap(), @@ -8634,6 +8635,10 @@ mod tests { panic!("expected JobFailed, got {err:?}"); }; assert_eq!(failure.message.as_deref(), Some("preempted")); + assert_eq!( + failure.traceback.as_deref(), + Some("Traceback (most recent call last):\nValueError: preempted") + ); assert_eq!(failure.phase.as_deref(), Some("commit")); assert_eq!(failure.retryable, Some(true)); assert_eq!(err.to_string(), "Job job-err failed: preempted (in commit)"); @@ -11029,7 +11034,8 @@ mod tests { "row_id": 42, "row_address": 12884901897, "column": "embedding", "function": "embed", "function_version": "2", "table_version": 11, "error_type": "ValueError", - "error_message": "bad input 'x'", "created_at_millis": 1700000000000}], + "error_message": "bad input 'x'", "traceback": "Traceback (most recent call last):\nValueError: bad input 'x'", + "created_at_millis": 1700000000000}], "fragments": [{"job_id": "j-7", "fragment_id": 4, "rows_skipped": 500, "rows_recorded": 100}], "truncated": true}"#, @@ -11060,6 +11066,9 @@ mod tests { table_version: 11, error_type: "ValueError".into(), error_message: "bad input 'x'".into(), + traceback: Some( + "Traceback (most recent call last):\nValueError: bad input 'x'".into() + ), created_at_millis: 1_700_000_000_000, }] ); @@ -11091,6 +11100,7 @@ mod tests { .unwrap(); assert!(legacy.row_id.is_none()); assert!(legacy.row_address.is_none()); + assert!(legacy.traceback.is_none()); // No filter sends no filter, and an empty listing reads as such. let table = Table::new_with_handler("my_table", |request| {