feat(client): support region request reuse and route refresh hints (#9540)

* feat(client): support shared region requests without deep cloning

Signed-off-by: WenyXu <wenymedia@gmail.com>

* feat: propagate route refresh retry hints

Signed-off-by: WenyXu <wenymedia@gmail.com>

* style: preserve local error variant imports in retry hints

Signed-off-by: WenyXu <wenymedia@gmail.com>

* fix(mito2): distinguish staging rejection from write backpressure

Signed-off-by: WenyXu <wenymedia@gmail.com>

* fix(meta): preserve legacy retry hints in instruction replies

Signed-off-by: WenyXu <wenymedia@gmail.com>

* test(client): inline retry hint compatibility cases

Signed-off-by: WenyXu <wenymedia@gmail.com>

* test(datanode): align batch-open retry hint assertion

Signed-off-by: WenyXu <wenymedia@gmail.com>

* style: use imported types in region request tests

Signed-off-by: WenyXu <wenymedia@gmail.com>

---------

Signed-off-by: WenyXu <wenymedia@gmail.com>

src/common/error/src/ext.rs

@@ -46,6 +46,10 @@ use crate::status_code::StatusCode;
 pub enum RetryHint {
     /// The operation may succeed if retried later.
     Retryable,
+    /// The operation may succeed after refreshing routing and partition information.
+    /// The request may need rebuilding when its partition assignment changes.
+    /// This hint does not imply that a failed operation had no side effects.
+    RetryAfterRouteRefresh,
     /// Retrying the same operation is not expected to help.
     ///
     /// This is the default for errors that do not explicitly opt in to retry.
@@ -53,16 +57,23 @@ pub enum RetryHint {
 }
 
 const RETRY_HINT_RETRYABLE: &str = "retryable";
+const RETRY_HINT_AFTER_ROUTE_REFRESH: &str = "retry_after_route_refresh";
 const RETRY_HINT_NON_RETRYABLE: &str = "non_retryable";
 
 impl RetryHint {
+    /// Returns whether another attempt may succeed, possibly after refreshing routes.
     pub fn is_retryable(self) -> bool {
-        matches!(self, RetryHint::Retryable)
+        matches!(
+            self,
+            RetryHint::Retryable | RetryHint::RetryAfterRouteRefresh
+        )
     }
 
+    /// Returns the retry hint encoded in gRPC error metadata.
     pub fn as_str(self) -> &'static str {
         match self {
             RetryHint::Retryable => RETRY_HINT_RETRYABLE,
+            RetryHint::RetryAfterRouteRefresh => RETRY_HINT_AFTER_ROUTE_REFRESH,
             RetryHint::NonRetryable => RETRY_HINT_NON_RETRYABLE,
         }
     }
@@ -90,6 +101,7 @@ impl FromStr for RetryHint {
     fn from_str(s: &str) -> Result<Self, Self::Err> {
         match s {
             RETRY_HINT_RETRYABLE => Ok(RetryHint::Retryable),
+            RETRY_HINT_AFTER_ROUTE_REFRESH => Ok(RetryHint::RetryAfterRouteRefresh),
             RETRY_HINT_NON_RETRYABLE => Ok(RetryHint::NonRetryable),
             _ => Err(()),
         }
@@ -133,7 +145,9 @@ pub trait ErrorExt: StackError {
     ///
     /// Implementations should return [`RetryHint::Retryable`] only when retrying the
     /// same operation may succeed without changing the request. The default is
-    /// [`RetryHint::NonRetryable`] to avoid accidental retry loops.
+    /// [`RetryHint::NonRetryable`] to avoid accidental retry loops. Use
+    /// [`RetryHint::RetryAfterRouteRefresh`] when routing or partition information
+    /// must be refreshed before another attempt.
     fn retry_hint(&self) -> RetryHint {
         RetryHint::NonRetryable
     }

src/common/error/tests/ext.rs

@@ -164,6 +164,15 @@ fn test_transparent_flag() {
 #[test]
 fn test_retry_hint_helpers() {
     assert!(RetryHint::Retryable.is_retryable());
+    assert!(RetryHint::RetryAfterRouteRefresh.is_retryable());
+    assert_eq!(
+        RetryHint::RetryAfterRouteRefresh.as_str(),
+        "retry_after_route_refresh"
+    );
+    assert_eq!(
+        RetryHint::from_str("retry_after_route_refresh").unwrap(),
+        RetryHint::RetryAfterRouteRefresh
+    );
     assert!(!RetryHint::NonRetryable.is_retryable());
 
     assert_eq!(RetryHint::Retryable.as_str(), "retryable");

src/common/meta/src/error.rs

@@ -1346,8 +1346,11 @@ impl ErrorExt for Error {
         use Error::*;
 
         match self {
-            RetryLater { .. }
-            | GetLatestCacheRetryExceeded { .. }
+            RetryLater { source, .. } => match source.retry_hint() {
+                RetryHint::RetryAfterRouteRefresh => RetryHint::RetryAfterRouteRefresh,
+                RetryHint::Retryable | RetryHint::NonRetryable => RetryHint::Retryable,
+            },
+            GetLatestCacheRetryExceeded { .. }
             | NoLeader { .. }
             | ElectionNoLeader { .. }
             | ElectionLeaderLeaseExpired { .. }

src/common/meta/src/instruction.rs

@@ -49,7 +49,7 @@ pub struct InstructionError {
     pub message: String,
     /// Retry hint serialized as a string.
     #[serde(
-        serialize_with = "RetryHint::serialize_as_str",
+        serialize_with = "InstructionError::serialize_retry_hint",
         deserialize_with = "RetryHint::deserialize_from_str"
     )]
     pub retry_hint: RetryHint,
@@ -108,6 +108,18 @@ impl InstructionError {
             retry_hint: error.retry_hint(),
         }
     }
+
+    /// Keeps instruction replies readable by peers that only know the legacy hints.
+    fn serialize_retry_hint<S>(hint: &RetryHint, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let hint = match hint {
+            RetryHint::RetryAfterRouteRefresh => RetryHint::Retryable,
+            hint => *hint,
+        };
+        RetryHint::serialize_as_str(&hint, serializer)
+    }
 }
 
 impl Display for InstructionError {
@@ -1450,6 +1462,26 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_instruction_error_route_refresh_compatibility() {
+        let error = InstructionError::new(
+            StatusCode::RegionNotFound,
+            "region not found",
+            RetryHint::RetryAfterRouteRefresh,
+        );
+        let serialized = serde_json::to_string(&error).unwrap();
+        assert_eq!(
+            r#"{"code":4005,"message":"region not found","retry_hint":"retryable"}"#,
+            serialized
+        );
+        let decoded: InstructionError = serde_json::from_str(&serialized).unwrap();
+        assert_eq!(decoded.code, error.code);
+        assert_eq!(decoded.message, error.message);
+        assert_eq!(decoded.retry_hint, RetryHint::Retryable);
+        assert_eq!(error.retry_hint, RetryHint::RetryAfterRouteRefresh);
+        assert_eq!(error.retry_hint.as_str(), "retry_after_route_refresh");
+    }
+
     #[test]
     fn test_instruction_error_from_error() {
         let error = MockError::new(StatusCode::RegionNotFound);