feat(procedure): Max retry time (#1095)

* feat: procedure config

* fix: modify config

* feat: add retry logic

* feat: add error

* feat: add it

* feat: add it

* feat: add it

* feat: rm retry from runner

* feat: use backon

* feat: add retry_interval

* feat: add retry_interval

* fix: conflict

* fix: cr

* feat: add retry error and id

* feat: rename

* refactor: execute

* feat: use config dir

* fix: cr

* fix: cr

* fix: fmt

* fix: fmt

* fix: pr

* fix: it

* fix: rm unless cmd params

* feat: add toml

* fix: ut

* feat: add rolling back

* fix: cr

* fix: cr

* fix: cr

* fix: ci

* fix: ci

* fix: ci

* chore: Apply suggestions from code review

---------

Co-authored-by: Yingwen <realevenyag@gmail.com>

src/table-procedure/src/create.rs

@@ -197,7 +197,7 @@ impl CreateTableProcedure {
         };
 
         match sub_state {
-            ProcedureState::Running => Ok(Status::Suspended {
+            ProcedureState::Running | ProcedureState::Retrying { .. } => Ok(Status::Suspended {
                 subprocedures: Vec::new(),
                 persist: false,
             }),

src/table-procedure/src/test_util.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use std::sync::Arc;
+use std::time::Duration;
 
 use catalog::local::MemoryCatalogManager;
 use catalog::CatalogManagerRef;
@@ -59,7 +60,11 @@ impl TestEnv {
         let accessor = Fs::default().root(&procedure_dir).build().unwrap();
         let object_store = ObjectStore::new(accessor).finish();
 
-        let procedure_manager = Arc::new(LocalManager::new(ManagerConfig { object_store }));
+        let procedure_manager = Arc::new(LocalManager::new(ManagerConfig {
+            object_store,
+            max_retry_times: 3,
+            retry_delay: Duration::from_secs(500),
+        }));
 
         let catalog_manager = Arc::new(MemoryCatalogManager::default());