rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-11-13 14:16:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: expose Function failure tracebacks (#4861)

Browse Source 
Function refresh failures and skipped-row diagnostics lose the traceback
when decoded by the SDK. Expose the server-provided optional `traceback`
on job failure details and Function error records in Rust, Python, and
Node.js, while keeping existing compact error messages.

Responses from older servers remain supported when the field is absent.
Tracebacks may be truncated by the server. The Rust `Error::JobFailed`
variant now stores `Box<JobFailure>` so the added diagnostics do not
enlarge every `Result` error payload.
This commit is contained in:
Xuanwo
2026-11-05 15:16:28 +08:00
committed by GitHub
parent b32afe9794
commit edc49e60e2
14 changed files with 82 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/src/js/interfaces/FunctionErrorRecord.md
+10
View File
@@ -90,3 +90,13 @@ detail was capped.
```ts
tableVersion: number;
```
***
### traceback?
```ts
optional traceback: string;
```
User-code traceback, possibly truncated by the server.
docs/src/js/interfaces/JobFailureInfo.md
+10
View File
@@ -31,3 +31,13 @@ optional phase: string;
```ts
optional retryable: boolean;
```
***
### traceback?
```ts
optional traceback: string;
```
User-code traceback, possibly truncated by the server.
nodejs/src/job.rs
+3
View File
@@ -141,6 +141,7 @@ impl Job {
self.inner.failure().map(|failure| JobFailureInfo {
phase: failure.phase,
message: failure.message,
traceback: failure.traceback,
retryable: failure.retryable,
})
}
@@ -218,5 +219,7 @@ impl From<lancedb::database::JobInfo> for JobInfo {
pub struct JobFailureInfo {
pub phase: Option<String>,
pub message: Option<String>,
/// User-code traceback, possibly truncated by the server.
pub traceback: Option<String>,
pub retryable: Option<bool>,
}
nodejs/src/table.rs
+3
View File
@@ -1530,6 +1530,8 @@ pub struct FunctionErrorRecord {
pub table_version: i64,
pub error_type: String,
pub error_message: String,
/// User-code traceback, possibly truncated by the server.
pub traceback: Option<String>,
pub created_at_millis: i64,
}
@@ -1545,6 +1547,7 @@ impl From<lancedb::function::FunctionErrorRecord> for FunctionErrorRecord {
table_version: record.table_version as i64,
error_type: record.error_type,
error_message: record.error_message,
traceback: record.traceback,
created_at_millis: record.created_at_millis,
}
}
python/python/lancedb/_lancedb.pyi
+6
View File
@@ -465,6 +465,10 @@ class JobFailureInfo:
@property
def message(self) -> Optional[str]: ...
@property
def traceback(self) -> Optional[str]:
"""User-code traceback, possibly truncated by the server."""
...
@property
def retryable(self) -> Optional[bool]: ...
@property
def sql_job_id(self) -> Optional[str]: ...
@@ -1014,6 +1018,8 @@ class FunctionErrorRecord:
table_version: int
error_type: str
error_message: str
traceback: Optional[str]
"""User-code traceback, possibly truncated by the server."""
created_at_millis: int
class FunctionErrorFragment:
python/python/tests/test_remote_db.py
+13
View File
@@ -2055,6 +2055,9 @@ def test_remote_function_errors_lists_the_rows_a_refresh_skipped():
"table_version": 11,
"error_type": "ValueError",
"error_message": "bad input 'x'",
"traceback": (
"Traceback (most recent call last):\nValueError: bad input 'x'"
),
"created_at_millis": 1700000000000,
}
],
@@ -2102,6 +2105,9 @@ def test_remote_function_errors_lists_the_rows_a_refresh_skipped():
assert errors.records[0].row_id == 42
assert errors.records[0].row_address == 12884901897
assert errors.records[0].function_version == "2"
assert errors.records[0].traceback == (
"Traceback (most recent call last):\nValueError: bad input 'x'"
)
assert repr(errors.records[0]) == (
'FunctionErrorRecord(job_id="j-7", fragment_id=3, row_offset=9, '
'row_address=12884901897, column="embedding", '
@@ -5128,6 +5134,10 @@ def test_remote_connection_jobs_surface():
failure=dict(
phase="execute",
message="worker died",
traceback=(
"Traceback (most recent call last):\n"
"RuntimeError: worker died"
),
retryable=True,
sql_job_id="sql-job-1",
),
@@ -5217,6 +5227,9 @@ def test_remote_connection_jobs_surface():
assert progress["waiting_for_job_id"] == "job-maintenance"
assert progress["paused_at_millis"] == 1200
assert job.failure.message == "worker died"
assert job.failure.traceback == (
"Traceback (most recent call last):\nRuntimeError: worker died"
)
assert job.failure.sql_job_id == "sql-job-1"
assert repr(job.failure) == (
'JobFailureInfo(phase="execute", message="worker died", retryable=True, '
python/src/job.rs
+3
View File
@@ -232,6 +232,8 @@ impl From<lancedb::database::JobInfo> for JobInfo {
pub struct JobFailureInfo {
phase: Option<String>,
message: Option<String>,
/// User-code traceback, possibly truncated by the server.
traceback: Option<String>,
retryable: Option<bool>,
sql_job_id: Option<String>,
}
@@ -342,6 +344,7 @@ impl From<lancedb::database::JobDescription> for JobDescription {
failure: description.failure.map(|failure| JobFailureInfo {
phase: failure.phase,
message: failure.message,
traceback: failure.traceback,
retryable: failure.retryable,
sql_job_id: failure.sql_job_id,
}),
python/src/table.rs
+3
View File
@@ -622,6 +622,8 @@ pub struct FunctionErrorRecord {
pub table_version: u64,
pub error_type: String,
pub error_message: String,
/// User-code traceback, possibly truncated by the server.
pub traceback: Option<String>,
pub created_at_millis: i64,
}
@@ -662,6 +664,7 @@ impl From<lancedb::function::FunctionErrorRecord> for FunctionErrorRecord {
table_version: record.table_version,
error_type: record.error_type,
error_message: record.error_message,
traceback: record.traceback,
created_at_millis: record.created_at_millis,
}
}
rust/lancedb/src/error.rs
+3 -1
View File
@@ -159,6 +159,8 @@ pub struct JobFailure {
pub phase: Option<String>,
/// A human-readable reason, when known.
pub message: Option<String>,
/// User-code traceback, possibly truncated by the server.
pub traceback: Option<String>,
/// Whether a retry could clear the failure, when known.
pub retryable: Option<bool>,
/// Isolated SQL job that raised this failure, when the server supplies
@@ -255,7 +257,7 @@ pub enum Error {
#[snafu(display("Job{} failed{failure}", job_id.as_ref().map(|id| format!(" {id}")).unwrap_or_default()))]
JobFailed {
job_id: Option<String>,
failure: JobFailure,
failure: Box<JobFailure>,
},
#[snafu(display("Job{} was cancelled", job_id.as_ref().map(|id| format!(" {id}")).unwrap_or_default()))]
JobCancelled { job_id: Option<String> },
rust/lancedb/src/function.rs
+3
View File
@@ -1121,6 +1121,9 @@ pub struct FunctionErrorRecord {
pub error_type: String,
/// The error's text.
pub error_message: String,
/// User-code traceback, possibly truncated by the server.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub traceback: Option<String>,
/// When the error was recorded, in milliseconds since the epoch.
pub created_at_millis: i64,
}
rust/lancedb/src/job.rs
+1 -1
View File
@@ -583,7 +583,7 @@ impl Outcome {
Self::Failed(source) => {
let (job_id, failure) = match source.as_ref() {
Error::JobFailed { job_id, failure } => (job_id.clone(), failure.clone()),
_ => (None, JobFailure::from_source(source)),
_ => (None, Box::new(JobFailure::from_source(source))),
};
Err(Error::JobFailed { job_id, failure })
}
rust/lancedb/src/remote/job.rs
+9 -4
View File
@@ -74,6 +74,8 @@ pub(super) struct ReportedFailure {
#[serde(default)]
message: Option<String>,
#[serde(default)]
traceback: Option<String>,
#[serde(default)]
retryable: Option<bool>,
#[serde(default)]
sql_job_id: Option<String>,
@@ -134,6 +136,7 @@ impl ReportedFailure {
JobFailure {
phase: self.phase,
message: self.message,
traceback: self.traceback,
retryable: self.retryable,
sql_job_id: self.sql_job_id,
source: None,
@@ -266,10 +269,12 @@ impl<S: HttpSend> JobHandle for RemoteJob<S> {
JobState::Failed => {
return Err(Error::JobFailed {
job_id: Some(self.job_id.clone()),
failure: description
.failure
.map(ReportedFailure::into_job_failure)
.unwrap_or_default(),
failure: Box::new(
description
.failure
.map(ReportedFailure::into_job_failure)
.unwrap_or_default(),
),
});
}
JobState::Cancelled => {
rust/lancedb/src/remote/sql.rs
+3 -2
View File
@@ -1666,13 +1666,14 @@ pub(super) fn sql_status_error(request_id: &str, status: &tonic::Status) -> Erro
if let Some(sql_job_id) = sql_job_id {
return Error::JobFailed {
job_id: Some(sql_job_id.clone()),
failure: JobFailure {
failure: Box::new(JobFailure {
phase: None,
message: Some(status.message().into()),
traceback: None,
retryable: None,
sql_job_id: Some(sql_job_id),
source: Some(Arc::new(error)),
},
}),
};
}
error
rust/lancedb/src/remote/table.rs
+12 -2
View File
@@ -8592,6 +8592,7 @@ mod tests {
// The server said only that it failed, so nothing may be invented.
assert!(failure.message.is_none(), "{failure:?}");
assert!(failure.phase.is_none(), "{failure:?}");
assert!(failure.traceback.is_none(), "{failure:?}");
assert!(failure.retryable.is_none(), "{failure:?}");
assert_eq!(err.to_string(), "Job job-err failed");
}
@@ -8616,7 +8617,7 @@ mod tests {
"/v1/jobs/describe" => http::Response::builder()
.status(200)
.body(
r#"{"job_id": "job-err", "job_state": "FAILED", "failure": {"phase": "commit", "message": "preempted", "retryable": true}}"#
r#"{"job_id": "job-err", "job_state": "FAILED", "failure": {"phase": "commit", "message": "preempted", "traceback": "Traceback (most recent call last):\nValueError: preempted", "retryable": true}}"#
.to_string(),
)
.unwrap(),
@@ -8634,6 +8635,10 @@ mod tests {
panic!("expected JobFailed, got {err:?}");
};
assert_eq!(failure.message.as_deref(), Some("preempted"));
assert_eq!(
failure.traceback.as_deref(),
Some("Traceback (most recent call last):\nValueError: preempted")
);
assert_eq!(failure.phase.as_deref(), Some("commit"));
assert_eq!(failure.retryable, Some(true));
assert_eq!(err.to_string(), "Job job-err failed: preempted (in commit)");
@@ -11029,7 +11034,8 @@ mod tests {
"row_id": 42, "row_address": 12884901897,
"column": "embedding", "function": "embed", "function_version": "2",
"table_version": 11, "error_type": "ValueError",
"error_message": "bad input 'x'", "created_at_millis": 1700000000000}],
"error_message": "bad input 'x'", "traceback": "Traceback (most recent call last):\nValueError: bad input 'x'",
"created_at_millis": 1700000000000}],
"fragments": [{"job_id": "j-7", "fragment_id": 4, "rows_skipped": 500,
"rows_recorded": 100}],
"truncated": true}"#,
@@ -11060,6 +11066,9 @@ mod tests {
table_version: 11,
error_type: "ValueError".into(),
error_message: "bad input 'x'".into(),
traceback: Some(
"Traceback (most recent call last):\nValueError: bad input 'x'".into()
),
created_at_millis: 1_700_000_000_000,
}]
);
@@ -11091,6 +11100,7 @@ mod tests {
.unwrap();
assert!(legacy.row_id.is_none());
assert!(legacy.row_address.is_none());
assert!(legacy.traceback.is_none());
// No filter sends no filter, and an empty listing reads as such.
let table = Table::new_with_handler("my_table", |request| {