feat: return per-entry region write failures (#9728)

* feat: return per-entry region write failures

Signed-off-by: WenyXu <wenymedia@gmail.com>

* fix: preserve retry hints for single-entry region writes

Signed-off-by: WenyXu <wenymedia@gmail.com>

* fix: preserve region write error compatibility

Signed-off-by: WenyXu <wenymedia@gmail.com>

* refactor: simplify region partial result handling

Signed-off-by: WenyXu <wenymedia@gmail.com>

* fix: align partial failure summaries with input order

Signed-off-by: WenyXu <wenymedia@gmail.com>

* refactor: separate region write response handling

Signed-off-by: WenyXu <wenymedia@gmail.com>

* test: fix partial result expectations and region map guard

Signed-off-by: WenyXu <wenymedia@gmail.com>

* chore: limit lockfile changes to greptime proto

Signed-off-by: WenyXu <wenymedia@gmail.com>

* fix: make partial region results an explicit client opt-in

Signed-off-by: WenyXu <wenymedia@gmail.com>

* chore: pin greptime-proto to merged partial-result changes

Signed-off-by: WenyXu <wenymedia@gmail.com>

---------

Signed-off-by: WenyXu <wenymedia@gmail.com>

src/common/error/src/status_code.rs

@@ -45,6 +45,8 @@ pub enum StatusCode {
     DeadlineExceeded = 1008,
     /// Service got suspended for various reason. For example, resources exceed limit.
     Suspended = 1009,
+    /// One or more entries failed; inspect per-entry results before retrying.
+    PartialFailure = 1010,
     // ====== End of common status code ================
 
     // ====== Begin of SQL related status code =========
@@ -150,7 +152,8 @@ impl StatusCode {
             | StatusCode::RuntimeResourcesExhausted
             | StatusCode::External => true,
 
-            StatusCode::Success
+            StatusCode::PartialFailure
+            | StatusCode::Success
             | StatusCode::Unsupported
             | StatusCode::InvalidArguments
             | StatusCode::InvalidSyntax
@@ -300,7 +303,7 @@ macro_rules! define_into_tonic_status {
 pub fn status_to_tonic_code(status_code: StatusCode) -> Code {
     match status_code {
         StatusCode::Success => Code::Ok,
-        StatusCode::Unknown | StatusCode::External => Code::Unknown,
+        StatusCode::Unknown | StatusCode::External | StatusCode::PartialFailure => Code::Unknown,
         StatusCode::Unsupported => Code::Unimplemented,
         StatusCode::Unexpected
         | StatusCode::IllegalState

src/common/meta/src/error.rs

@@ -41,6 +41,15 @@ pub use retry_hint::{retry_hint_from_postgres_error, retry_hint_from_postgres_po
 #[snafu(visibility(pub))]
 #[stack_trace_debug]
 pub enum Error {
+    #[snafu(display(
+        "Region request has {} failed entries; {}",
+        response.failures.len(),
+        response.first_failure().map(|(index, error)| format!("entry {index}: {}", error.output_msg())).unwrap_or_default()
+    ))]
+    PartialRegion {
+        response: Box<api::region::BatchRegionResponse<Error>>,
+    },
+
     #[snafu(display("Empty key is not allowed"))]
     EmptyKey {
         #[snafu(implicit)]
@@ -1195,6 +1204,9 @@ impl ErrorExt for Error {
     fn status_code(&self) -> StatusCode {
         use Error::*;
         match self {
+            Error::PartialRegion { response } => response
+                .first_failure()
+                .map_or(StatusCode::Internal, |(_, error)| error.status_code()),
             IllegalServerState { .. }
             | EtcdTxnOpResponse { .. }
             | EtcdFailed { .. }
@@ -1361,6 +1373,7 @@ impl ErrorExt for Error {
         use Error::*;
 
         match self {
+            Error::PartialRegion { .. } => RetryHint::Retryable,
             RetryLater { source, .. } => match source.retry_hint() {
                 RetryHint::RetryAfterRouteRefresh => RetryHint::RetryAfterRouteRefresh,
                 RetryHint::Retryable | RetryHint::NonRetryable => RetryHint::Retryable,