1use std::str::Utf8Error;
16use std::sync::Arc;
17
18use common_error::ext::{BoxedError, ErrorExt, RetryHint};
19use common_error::status_code::StatusCode;
20use common_macro::stack_trace_debug;
21use common_procedure::ProcedureId;
22use common_wal::kafka::rskafka_client_error_to_retry_hint;
23use object_store::error::retry_hint_from_opendal_error;
24use serde_json::error::Error as JsonError;
25use snafu::{Location, Snafu};
26use store_api::storage::RegionId;
27use table::metadata::TableId;
28
29use crate::DatanodeId;
30use crate::peer::Peer;
31
32mod retry_hint;
33
34pub use retry_hint::retry_hint_from_etcd_error;
35#[cfg(feature = "mysql_kvbackend")]
36pub use retry_hint::retry_hint_from_sqlx_error;
37#[cfg(feature = "pg_kvbackend")]
38pub use retry_hint::{retry_hint_from_postgres_error, retry_hint_from_postgres_pool_error};
39
40#[derive(Snafu)]
41#[snafu(visibility(pub))]
42#[stack_trace_debug]
43pub enum Error {
44 #[snafu(display("Failed to execute batch request"))]
45 BatchRequest { source: Arc<Error> },
46
47 #[snafu(display(
48 "Region request has {} failed entries; {}",
49 response.failures.len(),
50 response.first_failure().map(|(index, error)| format!("entry {index}: {}", error.output_msg())).unwrap_or_default()
51 ))]
52 PartialRegion {
53 response: Box<api::region::BatchRegionResponse<Error>>,
54 },
55
56 #[snafu(display("Empty key is not allowed"))]
57 EmptyKey {
58 #[snafu(implicit)]
59 location: Location,
60 },
61
62 #[snafu(display(
63 "Another procedure is operating the region: {} on peer: {}",
64 region_id,
65 peer_id
66 ))]
67 RegionOperatingRace {
68 #[snafu(implicit)]
69 location: Location,
70 peer_id: DatanodeId,
71 region_id: RegionId,
72 },
73
74 #[snafu(display("Failed to connect to Etcd"))]
75 ConnectEtcd {
76 #[snafu(source)]
77 error: etcd_client::Error,
78 #[snafu(implicit)]
79 location: Location,
80 },
81
82 #[snafu(display("Failed to execute via Etcd"))]
83 EtcdFailed {
84 #[snafu(source)]
85 error: etcd_client::Error,
86 #[snafu(implicit)]
87 location: Location,
88 },
89
90 #[snafu(display("Failed to execute {} txn operations via Etcd", max_operations))]
91 EtcdTxnFailed {
92 max_operations: usize,
93 #[snafu(source)]
94 error: etcd_client::Error,
95 #[snafu(implicit)]
96 location: Location,
97 },
98
99 #[snafu(display("Failed to get sequence: {}", err_msg))]
100 NextSequence {
101 err_msg: String,
102 #[snafu(implicit)]
103 location: Location,
104 },
105
106 #[snafu(display("Unexpected sequence value: {}", err_msg))]
107 UnexpectedSequenceValue {
108 err_msg: String,
109 #[snafu(implicit)]
110 location: Location,
111 },
112
113 #[snafu(display("Table info not found: {}", table))]
114 TableInfoNotFound {
115 table: String,
116 #[snafu(implicit)]
117 location: Location,
118 },
119
120 #[snafu(display("Failed to register procedure loader, type name: {}", type_name))]
121 RegisterProcedureLoader {
122 type_name: String,
123 #[snafu(implicit)]
124 location: Location,
125 source: common_procedure::error::Error,
126 },
127
128 #[snafu(display("Failed to register repartition procedure loader"))]
129 RegisterRepartitionProcedureLoader {
130 #[snafu(implicit)]
131 location: Location,
132 source: BoxedError,
133 },
134
135 #[snafu(display("Failed to create repartition procedure"))]
136 CreateRepartitionProcedure {
137 source: BoxedError,
138 #[snafu(implicit)]
139 location: Location,
140 },
141
142 #[snafu(display("Failed to persist the repartition GC requirement"))]
143 PersistRepartitionGcRequirement {
144 source: BoxedError,
145 #[snafu(implicit)]
146 location: Location,
147 },
148
149 #[snafu(display("Failed to submit procedure"))]
150 SubmitProcedure {
151 #[snafu(implicit)]
152 location: Location,
153 source: common_procedure::Error,
154 },
155
156 #[snafu(display("Failed to query procedure"))]
157 QueryProcedure {
158 #[snafu(implicit)]
159 location: Location,
160 source: common_procedure::Error,
161 },
162
163 #[snafu(display("Procedure not found: {pid}"))]
164 ProcedureNotFound {
165 #[snafu(implicit)]
166 location: Location,
167 pid: String,
168 },
169
170 #[snafu(display("Failed to parse procedure id: {key}"))]
171 ParseProcedureId {
172 #[snafu(implicit)]
173 location: Location,
174 key: String,
175 #[snafu(source)]
176 error: common_procedure::ParseIdError,
177 },
178
179 #[snafu(display("Unsupported operation {}", operation))]
180 Unsupported {
181 operation: String,
182 #[snafu(implicit)]
183 location: Location,
184 },
185
186 #[snafu(display("Trying to write to a read-only kv backend: {}", name))]
187 ReadOnlyKvBackend {
188 name: String,
189 #[snafu(implicit)]
190 location: Location,
191 },
192
193 #[snafu(display("Failed to get procedure state receiver, procedure id: {procedure_id}"))]
194 ProcedureStateReceiver {
195 procedure_id: ProcedureId,
196 #[snafu(implicit)]
197 location: Location,
198 source: common_procedure::Error,
199 },
200
201 #[snafu(display("Procedure state receiver not found: {procedure_id}"))]
202 ProcedureStateReceiverNotFound {
203 procedure_id: ProcedureId,
204 #[snafu(implicit)]
205 location: Location,
206 },
207
208 #[snafu(display("Failed to wait procedure done"))]
209 WaitProcedure {
210 #[snafu(implicit)]
211 location: Location,
212 source: common_procedure::Error,
213 },
214
215 #[snafu(display(
216 "DDL request timed out after {timeout:?}; a submitted procedure may still be running"
217 ))]
218 DdlTaskTimeout {
219 timeout: std::time::Duration,
220 #[snafu(implicit)]
221 location: Location,
222 },
223
224 #[snafu(display("Failed to start procedure manager"))]
225 StartProcedureManager {
226 #[snafu(implicit)]
227 location: Location,
228 source: common_procedure::Error,
229 },
230
231 #[snafu(display("Failed to stop procedure manager"))]
232 StopProcedureManager {
233 #[snafu(implicit)]
234 location: Location,
235 source: common_procedure::Error,
236 },
237
238 #[snafu(display(
239 "Failed to get procedure output, procedure id: {procedure_id}, error: {err_msg}"
240 ))]
241 ProcedureOutput {
242 procedure_id: String,
243 err_msg: String,
244 #[snafu(implicit)]
245 location: Location,
246 },
247
248 #[snafu(display("Primary key '{key}' not found when creating region request"))]
249 PrimaryKeyNotFound {
250 key: String,
251 #[snafu(implicit)]
252 location: Location,
253 },
254
255 #[snafu(display("Failed to build table meta for table: {}", table_name))]
256 BuildTableMeta {
257 table_name: String,
258 #[snafu(source)]
259 error: table::metadata::TableMetaBuilderError,
260 #[snafu(implicit)]
261 location: Location,
262 },
263
264 #[snafu(display("Table occurs error"))]
265 Table {
266 #[snafu(implicit)]
267 location: Location,
268 source: table::error::Error,
269 },
270
271 #[snafu(display("Failed to find table route for table id {}", table_id))]
272 TableRouteNotFound {
273 table_id: TableId,
274 #[snafu(implicit)]
275 location: Location,
276 },
277
278 #[snafu(display("Failed to find table repartition metadata for table id {}", table_id))]
279 TableRepartNotFound {
280 table_id: TableId,
281 #[snafu(implicit)]
282 location: Location,
283 },
284
285 #[snafu(display("Failed to decode protobuf"))]
286 DecodeProto {
287 #[snafu(implicit)]
288 location: Location,
289 #[snafu(source)]
290 error: prost::DecodeError,
291 },
292
293 #[snafu(display("Failed to decode packed file references"))]
294 DecodePackedFileRefs {
295 #[snafu(implicit)]
296 location: Location,
297 #[snafu(source)]
298 error: base64::DecodeError,
299 },
300
301 #[snafu(display("Invalid packed file references framing"))]
302 InvalidPackedFileRefs {
303 #[snafu(implicit)]
304 location: Location,
305 },
306
307 #[snafu(display("Failed to encode object into json"))]
308 EncodeJson {
309 #[snafu(implicit)]
310 location: Location,
311 #[snafu(source)]
312 error: JsonError,
313 },
314
315 #[snafu(display("Failed to decode object from json"))]
316 DecodeJson {
317 #[snafu(implicit)]
318 location: Location,
319 #[snafu(source)]
320 error: JsonError,
321 },
322
323 #[snafu(display("Failed to serialize to json: {}", input))]
324 SerializeToJson {
325 input: String,
326 #[snafu(source)]
327 error: serde_json::error::Error,
328 #[snafu(implicit)]
329 location: Location,
330 },
331
332 #[snafu(display("Failed to deserialize from json: {}", input))]
333 DeserializeFromJson {
334 input: String,
335 #[snafu(source)]
336 error: serde_json::error::Error,
337 #[snafu(implicit)]
338 location: Location,
339 },
340
341 #[snafu(display("Payload not exist"))]
342 PayloadNotExist {
343 #[snafu(implicit)]
344 location: Location,
345 },
346
347 #[snafu(display("Failed to serde json"))]
348 SerdeJson {
349 #[snafu(source)]
350 error: serde_json::error::Error,
351 #[snafu(implicit)]
352 location: Location,
353 },
354
355 #[snafu(display("Failed to parse value {} into key {}", value, key))]
356 ParseOption {
357 key: String,
358 value: String,
359 #[snafu(implicit)]
360 location: Location,
361 },
362
363 #[snafu(display(
364 "Conflicting schema options: {}={} and {}={}",
365 first_key,
366 first_value,
367 second_key,
368 second_value
369 ))]
370 ConflictingSchemaOptions {
371 first_key: String,
372 first_value: String,
373 second_key: String,
374 second_value: String,
375 #[snafu(implicit)]
376 location: Location,
377 },
378
379 #[snafu(display("Illegal state from server, code: {}, error: {}", code, err_msg))]
380 IllegalServerState {
381 code: i32,
382 err_msg: String,
383 #[snafu(implicit)]
384 location: Location,
385 },
386
387 #[snafu(display("Failed to convert alter table request"))]
388 ConvertAlterTableRequest {
389 source: common_grpc_expr::error::Error,
390 #[snafu(implicit)]
391 location: Location,
392 },
393
394 #[snafu(display("Invalid protobuf message: {err_msg}"))]
395 InvalidProtoMsg {
396 err_msg: String,
397 #[snafu(implicit)]
398 location: Location,
399 },
400
401 #[snafu(display("Unexpected: {err_msg}"))]
402 Unexpected {
403 err_msg: String,
404 #[snafu(implicit)]
405 location: Location,
406 },
407
408 #[snafu(display("Metasrv election has no leader at this moment"))]
409 ElectionNoLeader {
410 #[snafu(implicit)]
411 location: Location,
412 },
413
414 #[snafu(display("Metasrv election leader lease expired"))]
415 ElectionLeaderLeaseExpired {
416 #[snafu(implicit)]
417 location: Location,
418 },
419
420 #[snafu(display("Metasrv election leader lease changed during election"))]
421 ElectionLeaderLeaseChanged {
422 #[snafu(implicit)]
423 location: Location,
424 },
425
426 #[snafu(display("Table already exists, table: {}", table_name))]
427 TableAlreadyExists {
428 table_name: String,
429 #[snafu(implicit)]
430 location: Location,
431 },
432
433 #[snafu(display(
434 "Cannot drop table '{}': an older tombstone already uses the same full name",
435 table_name
436 ))]
437 TableNameTombstoneConflict {
439 table_name: String,
440 existing_table_id: TableId,
441 dropping_table_id: TableId,
442 #[snafu(implicit)]
443 location: Location,
444 },
445
446 #[snafu(display("View already exists, view: {}", view_name))]
447 ViewAlreadyExists {
448 view_name: String,
449 #[snafu(implicit)]
450 location: Location,
451 },
452
453 #[snafu(display("Flow already exists: {}", flow_name))]
454 FlowAlreadyExists {
455 flow_name: String,
456 #[snafu(implicit)]
457 location: Location,
458 },
459
460 #[snafu(display("Schema already exists, catalog:{}, schema: {}", catalog, schema))]
461 SchemaAlreadyExists {
462 catalog: String,
463 schema: String,
464 #[snafu(implicit)]
465 location: Location,
466 },
467
468 #[snafu(display("Failed to convert raw key to str"))]
469 ConvertRawKey {
470 #[snafu(implicit)]
471 location: Location,
472 #[snafu(source)]
473 error: Utf8Error,
474 },
475
476 #[snafu(display("Table not found: '{}'", table_name))]
477 TableNotFound {
478 table_name: String,
479 #[snafu(implicit)]
480 location: Location,
481 },
482
483 #[snafu(display("Region not found: {}", region_id))]
484 RegionNotFound {
485 region_id: RegionId,
486 #[snafu(implicit)]
487 location: Location,
488 },
489
490 #[snafu(display("View not found: '{}'", view_name))]
491 ViewNotFound {
492 view_name: String,
493 #[snafu(implicit)]
494 location: Location,
495 },
496
497 #[snafu(display("Flow not found: '{}'", flow_name))]
498 FlowNotFound {
499 flow_name: String,
500 #[snafu(implicit)]
501 location: Location,
502 },
503
504 #[snafu(display("Flow route not found: '{}'", flow_name))]
505 FlowRouteNotFound {
506 flow_name: String,
507 #[snafu(implicit)]
508 location: Location,
509 },
510
511 #[snafu(display("Schema nod found, schema: {}", table_schema))]
512 SchemaNotFound {
513 table_schema: String,
514 #[snafu(implicit)]
515 location: Location,
516 },
517
518 #[snafu(display("Catalog not found, catalog: {}", catalog))]
519 CatalogNotFound {
520 catalog: String,
521 #[snafu(implicit)]
522 location: Location,
523 },
524
525 #[snafu(display("Invalid metadata, err: {}", err_msg))]
526 InvalidMetadata {
527 err_msg: String,
528 #[snafu(implicit)]
529 location: Location,
530 },
531
532 #[snafu(display("Invalid view info, err: {}", err_msg))]
533 InvalidViewInfo {
534 err_msg: String,
535 #[snafu(implicit)]
536 location: Location,
537 },
538
539 #[snafu(display("Invalid SQL, error: {err_msg}"))]
540 InvalidFlowOption {
541 err_msg: String,
542 #[snafu(implicit)]
543 location: Location,
544 },
545
546 #[snafu(display("Invalid flow request body: {:?}", body))]
547 InvalidFlowRequestBody {
548 body: Box<Option<api::v1::flow::flow_request::Body>>,
549 #[snafu(implicit)]
550 location: Location,
551 },
552
553 #[snafu(display("Failed to get kv cache, err: {}", err_msg))]
554 GetKvCache { err_msg: String },
555
556 #[snafu(display("Get null from cache, key: {}", key))]
557 CacheNotGet {
558 key: String,
559 #[snafu(implicit)]
560 location: Location,
561 },
562
563 #[snafu(display("Etcd txn error: {err_msg}"))]
564 EtcdTxnOpResponse {
565 err_msg: String,
566 #[snafu(implicit)]
567 location: Location,
568 },
569
570 #[snafu(display("External error"))]
571 External {
572 #[snafu(implicit)]
573 location: Location,
574 source: BoxedError,
575 },
576
577 #[snafu(display("The response exceeded size limit"))]
578 ResponseExceededSizeLimit {
579 #[snafu(implicit)]
580 location: Location,
581 source: BoxedError,
582 },
583
584 #[snafu(display("Invalid heartbeat response"))]
585 InvalidHeartbeatResponse {
586 #[snafu(implicit)]
587 location: Location,
588 },
589
590 #[snafu(display("Failed to operate on datanode: {}", peer))]
591 OperateDatanode {
592 #[snafu(implicit)]
593 location: Location,
594 peer: Peer,
595 source: BoxedError,
596 },
597
598 #[snafu(display("Retry later"))]
599 RetryLater {
600 source: BoxedError,
601 clean_poisons: bool,
602 },
603
604 #[snafu(display("Abort procedure"))]
605 AbortProcedure {
606 #[snafu(implicit)]
607 location: Location,
608 source: BoxedError,
609 clean_poisons: bool,
610 },
611
612 #[snafu(display("Failed to serialize WAL options for region: {region_id}"))]
613 SerializeWalOptions {
614 region_id: RegionId,
615 #[snafu(source)]
616 error: serde_json::Error,
617 #[snafu(implicit)]
618 location: Location,
619 },
620
621 #[snafu(display("Invalid number of topics {}", num_topics))]
622 InvalidNumTopics {
623 num_topics: usize,
624 #[snafu(implicit)]
625 location: Location,
626 },
627
628 #[snafu(display(
629 "Failed to build a Kafka client, broker endpoints: {:?}",
630 broker_endpoints
631 ))]
632 BuildKafkaClient {
633 broker_endpoints: Vec<String>,
634 #[snafu(implicit)]
635 location: Location,
636 #[snafu(source)]
637 error: rskafka::client::error::Error,
638 },
639
640 #[snafu(display("Failed to create TLS Config"))]
641 TlsConfig {
642 #[snafu(implicit)]
643 location: Location,
644 source: common_wal::error::Error,
645 },
646
647 #[snafu(display("Failed to build a Kafka controller client"))]
648 BuildKafkaCtrlClient {
649 #[snafu(implicit)]
650 location: Location,
651 #[snafu(source)]
652 error: rskafka::client::error::Error,
653 },
654
655 #[snafu(display(
656 "Failed to get a Kafka partition client, topic: {}, partition: {}",
657 topic,
658 partition
659 ))]
660 KafkaPartitionClient {
661 topic: String,
662 partition: i32,
663 #[snafu(implicit)]
664 location: Location,
665 #[snafu(source)]
666 error: rskafka::client::error::Error,
667 },
668
669 #[snafu(display(
670 "Failed to get offset from Kafka, topic: {}, partition: {}",
671 topic,
672 partition
673 ))]
674 KafkaGetOffset {
675 topic: String,
676 partition: i32,
677 #[snafu(implicit)]
678 location: Location,
679 #[snafu(source)]
680 error: rskafka::client::error::Error,
681 },
682
683 #[snafu(display("Failed to produce records to Kafka, topic: {}", topic))]
684 ProduceRecord {
685 topic: String,
686 #[snafu(implicit)]
687 location: Location,
688 #[snafu(source)]
689 error: rskafka::client::error::Error,
690 },
691
692 #[snafu(display("Failed to create a Kafka wal topic"))]
693 CreateKafkaWalTopic {
694 #[snafu(implicit)]
695 location: Location,
696 #[snafu(source)]
697 error: rskafka::client::error::Error,
698 },
699
700 #[snafu(display("The topic pool is empty"))]
701 EmptyTopicPool {
702 #[snafu(implicit)]
703 location: Location,
704 },
705
706 #[snafu(display("Unexpected table route type: {}", err_msg))]
707 UnexpectedLogicalRouteTable {
708 #[snafu(implicit)]
709 location: Location,
710 err_msg: String,
711 },
712
713 #[snafu(display("The tasks of {} cannot be empty", name))]
714 EmptyDdlTasks {
715 name: String,
716 #[snafu(implicit)]
717 location: Location,
718 },
719
720 #[snafu(display("Metadata corruption: {}", err_msg))]
721 MetadataCorruption {
722 err_msg: String,
723 #[snafu(implicit)]
724 location: Location,
725 },
726
727 #[snafu(display("Alter logical tables invalid arguments: {}", err_msg))]
728 AlterLogicalTablesInvalidArguments {
729 err_msg: String,
730 #[snafu(implicit)]
731 location: Location,
732 },
733
734 #[snafu(display("Create logical tables invalid arguments: {}", err_msg))]
735 CreateLogicalTablesInvalidArguments {
736 err_msg: String,
737 #[snafu(implicit)]
738 location: Location,
739 },
740
741 #[snafu(display("Invalid node info key: {}", key))]
742 InvalidNodeInfoKey {
743 key: String,
744 #[snafu(implicit)]
745 location: Location,
746 },
747
748 #[snafu(display("Invalid node stat key: {}", key))]
749 InvalidStatKey {
750 key: String,
751 #[snafu(implicit)]
752 location: Location,
753 },
754
755 #[snafu(display("Failed to parse number: {}", err_msg))]
756 ParseNum {
757 err_msg: String,
758 #[snafu(source)]
759 error: std::num::ParseIntError,
760 #[snafu(implicit)]
761 location: Location,
762 },
763
764 #[snafu(display("Invalid role: {}", role))]
765 InvalidRole {
766 role: i32,
767 #[snafu(implicit)]
768 location: Location,
769 },
770
771 #[snafu(display("Invalid set database option, key: {}, value: {}", key, value))]
772 InvalidSetDatabaseOption {
773 key: String,
774 value: String,
775 #[snafu(implicit)]
776 location: Location,
777 },
778
779 #[snafu(display("Invalid unset database option, key: {}", key))]
780 InvalidUnsetDatabaseOption {
781 key: String,
782 #[snafu(implicit)]
783 location: Location,
784 },
785
786 #[snafu(display("Invalid prefix: {}, key: {}", prefix, key))]
787 MismatchPrefix {
788 prefix: String,
789 key: String,
790 #[snafu(implicit)]
791 location: Location,
792 },
793
794 #[snafu(display("Failed to move values: {err_msg}"))]
795 MoveValues {
796 err_msg: String,
797 #[snafu(implicit)]
798 location: Location,
799 },
800
801 #[snafu(display("Failed to restore tombstone, target key already exists: {key}"))]
802 TombstoneTargetAlreadyExists {
803 key: String,
804 #[snafu(implicit)]
805 location: Location,
806 },
807
808 #[snafu(display("Failed to parse {} from utf8", name))]
809 FromUtf8 {
810 name: String,
811 #[snafu(source)]
812 error: std::string::FromUtf8Error,
813 #[snafu(implicit)]
814 location: Location,
815 },
816
817 #[snafu(display("Value not exists"))]
818 ValueNotExist {
819 #[snafu(implicit)]
820 location: Location,
821 },
822
823 #[snafu(display("Failed to get cache"))]
824 GetCache { source: Arc<Error> },
825
826 #[snafu(display(
827 "Failed to get latest cache value after {} attempts due to concurrent invalidation",
828 attempts
829 ))]
830 GetLatestCacheRetryExceeded {
831 attempts: usize,
832 #[snafu(implicit)]
833 location: Location,
834 },
835
836 #[cfg(feature = "pg_kvbackend")]
837 #[snafu(display("Failed to execute via Postgres, sql: {}", sql))]
838 PostgresExecution {
839 sql: String,
840 #[snafu(source)]
841 error: tokio_postgres::Error,
842 #[snafu(implicit)]
843 location: Location,
844 },
845
846 #[cfg(feature = "pg_kvbackend")]
847 #[snafu(display("Failed to create connection pool for Postgres"))]
848 CreatePostgresPool {
849 #[snafu(source)]
850 error: deadpool_postgres::CreatePoolError,
851 #[snafu(implicit)]
852 location: Location,
853 },
854
855 #[cfg(feature = "pg_kvbackend")]
856 #[snafu(display("Failed to get Postgres connection from pool: {}", reason))]
857 GetPostgresConnection {
858 reason: String,
859 #[snafu(implicit)]
860 location: Location,
861 },
862
863 #[cfg(feature = "pg_kvbackend")]
864 #[snafu(display("Failed to get Postgres client"))]
865 GetPostgresClient {
866 #[snafu(source)]
867 error: deadpool::managed::PoolError<tokio_postgres::Error>,
868 #[snafu(implicit)]
869 location: Location,
870 },
871
872 #[cfg(feature = "pg_kvbackend")]
873 #[snafu(display("Failed to {} Postgres transaction", operation))]
874 PostgresTransaction {
875 #[snafu(source)]
876 error: tokio_postgres::Error,
877 #[snafu(implicit)]
878 location: Location,
879 operation: String,
880 },
881
882 #[cfg(feature = "pg_kvbackend")]
883 #[snafu(display("Failed to setup PostgreSQL TLS configuration: {}", reason))]
884 PostgresTlsConfig {
885 reason: String,
886 #[snafu(implicit)]
887 location: Location,
888 },
889
890 #[snafu(display("Failed to load TLS certificate from path: {}", path))]
891 LoadTlsCertificate {
892 path: String,
893 #[snafu(source)]
894 error: std::io::Error,
895 #[snafu(implicit)]
896 location: Location,
897 },
898
899 #[cfg(feature = "pg_kvbackend")]
900 #[snafu(display("Invalid TLS configuration: {}", reason))]
901 InvalidTlsConfig {
902 reason: String,
903 #[snafu(implicit)]
904 location: Location,
905 },
906
907 #[cfg(feature = "mysql_kvbackend")]
908 #[snafu(display("Failed to execute via MySql, sql: {}", sql))]
909 MySqlExecution {
910 sql: String,
911 #[snafu(source)]
912 error: sqlx::Error,
913 #[snafu(implicit)]
914 location: Location,
915 },
916
917 #[cfg(feature = "mysql_kvbackend")]
918 #[snafu(display("Failed to create connection pool for MySql"))]
919 CreateMySqlPool {
920 #[snafu(source)]
921 error: sqlx::Error,
922 #[snafu(implicit)]
923 location: Location,
924 },
925
926 #[cfg(feature = "mysql_kvbackend")]
927 #[snafu(display("Failed to decode sql value"))]
928 DecodeSqlValue {
929 #[snafu(source)]
930 error: sqlx::error::Error,
931 #[snafu(implicit)]
932 location: Location,
933 },
934
935 #[cfg(feature = "mysql_kvbackend")]
936 #[snafu(display("Failed to acquire mysql client from pool"))]
937 AcquireMySqlClient {
938 #[snafu(source)]
939 error: sqlx::Error,
940 #[snafu(implicit)]
941 location: Location,
942 },
943
944 #[cfg(feature = "mysql_kvbackend")]
945 #[snafu(display("Failed to {} MySql transaction", operation))]
946 MySqlTransaction {
947 #[snafu(source)]
948 error: sqlx::Error,
949 #[snafu(implicit)]
950 location: Location,
951 operation: String,
952 },
953
954 #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
955 #[snafu(display("Rds transaction retry failed"))]
956 RdsTransactionRetryFailed {
957 #[snafu(implicit)]
958 location: Location,
959 },
960
961 #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
962 #[snafu(display("Sql execution timeout, sql: {}, duration: {:?}", sql, duration))]
963 SqlExecutionTimeout {
964 sql: String,
965 duration: std::time::Duration,
966 #[snafu(implicit)]
967 location: Location,
968 },
969
970 #[snafu(display(
971 "Datanode table info not found, table id: {}, datanode id: {}",
972 table_id,
973 datanode_id
974 ))]
975 DatanodeTableInfoNotFound {
976 datanode_id: DatanodeId,
977 table_id: TableId,
978 #[snafu(implicit)]
979 location: Location,
980 },
981
982 #[snafu(display("Invalid topic name prefix: {}", prefix))]
983 InvalidTopicNamePrefix {
984 prefix: String,
985 #[snafu(implicit)]
986 location: Location,
987 },
988
989 #[snafu(display("No leader found for table_id: {}", table_id))]
990 NoLeader {
991 table_id: TableId,
992 #[snafu(implicit)]
993 location: Location,
994 },
995
996 #[snafu(display(
997 "Procedure poison key already exists with a different value, key: {}, value: {}",
998 key,
999 value
1000 ))]
1001 ProcedurePoisonConflict {
1002 key: String,
1003 value: String,
1004 #[snafu(implicit)]
1005 location: Location,
1006 },
1007
1008 #[snafu(display("Failed to put poison, table metadata may be corrupted"))]
1009 PutPoison {
1010 #[snafu(implicit)]
1011 location: Location,
1012 #[snafu(source)]
1013 source: common_procedure::error::Error,
1014 },
1015
1016 #[snafu(display("Invalid file path: {}", file_path))]
1017 InvalidFilePath {
1018 #[snafu(implicit)]
1019 location: Location,
1020 file_path: String,
1021 },
1022
1023 #[snafu(display("Failed to serialize flexbuffers"))]
1024 SerializeFlexbuffers {
1025 #[snafu(implicit)]
1026 location: Location,
1027 #[snafu(source)]
1028 error: flexbuffers::SerializationError,
1029 },
1030
1031 #[snafu(display("Failed to deserialize flexbuffers"))]
1032 DeserializeFlexbuffers {
1033 #[snafu(implicit)]
1034 location: Location,
1035 #[snafu(source)]
1036 error: flexbuffers::DeserializationError,
1037 },
1038
1039 #[snafu(display("Failed to read flexbuffers"))]
1040 ReadFlexbuffers {
1041 #[snafu(implicit)]
1042 location: Location,
1043 #[snafu(source)]
1044 error: flexbuffers::ReaderError,
1045 },
1046
1047 #[snafu(display("Invalid file name: {}", reason))]
1048 InvalidFileName {
1049 #[snafu(implicit)]
1050 location: Location,
1051 reason: String,
1052 },
1053
1054 #[snafu(display("Invalid file extension: {}", reason))]
1055 InvalidFileExtension {
1056 #[snafu(implicit)]
1057 location: Location,
1058 reason: String,
1059 },
1060
1061 #[snafu(display("Failed to write object, file path: {}", file_path))]
1062 WriteObject {
1063 #[snafu(implicit)]
1064 location: Location,
1065 file_path: String,
1066 #[snafu(source)]
1067 error: object_store::Error,
1068 },
1069
1070 #[snafu(display("Failed to read object, file path: {}", file_path))]
1071 ReadObject {
1072 #[snafu(implicit)]
1073 location: Location,
1074 file_path: String,
1075 #[snafu(source)]
1076 error: object_store::Error,
1077 },
1078
1079 #[snafu(display("Missing column ids"))]
1080 MissingColumnIds {
1081 #[snafu(implicit)]
1082 location: Location,
1083 },
1084
1085 #[snafu(display(
1086 "Missing column in column metadata: {}, table: {}, table_id: {}",
1087 column_name,
1088 table_name,
1089 table_id,
1090 ))]
1091 MissingColumnInColumnMetadata {
1092 column_name: String,
1093 #[snafu(implicit)]
1094 location: Location,
1095 table_name: String,
1096 table_id: TableId,
1097 },
1098
1099 #[snafu(display(
1100 "Mismatch column id: column_name: {}, column_id: {}, table: {}, table_id: {}",
1101 column_name,
1102 column_id,
1103 table_name,
1104 table_id,
1105 ))]
1106 MismatchColumnId {
1107 column_name: String,
1108 column_id: u32,
1109 #[snafu(implicit)]
1110 location: Location,
1111 table_name: String,
1112 table_id: TableId,
1113 },
1114
1115 #[snafu(display("Failed to convert column def, column: {}", column))]
1116 ConvertColumnDef {
1117 column: String,
1118 #[snafu(implicit)]
1119 location: Location,
1120 source: api::error::Error,
1121 },
1122
1123 #[snafu(display("Failed to convert time ranges"))]
1124 ConvertTimeRanges {
1125 #[snafu(implicit)]
1126 location: Location,
1127 source: api::error::Error,
1128 },
1129
1130 #[snafu(display(
1131 "Column metadata inconsistencies found in table: {}, table_id: {}",
1132 table_name,
1133 table_id
1134 ))]
1135 ColumnMetadataConflicts {
1136 table_name: String,
1137 table_id: TableId,
1138 },
1139
1140 #[snafu(display(
1141 "Column not found in column metadata, column_name: {}, column_id: {}",
1142 column_name,
1143 column_id
1144 ))]
1145 ColumnNotFound { column_name: String, column_id: u32 },
1146
1147 #[snafu(display(
1148 "Column id mismatch, column_name: {}, expected column_id: {}, actual column_id: {}",
1149 column_name,
1150 expected_column_id,
1151 actual_column_id
1152 ))]
1153 ColumnIdMismatch {
1154 column_name: String,
1155 expected_column_id: u32,
1156 actual_column_id: u32,
1157 },
1158
1159 #[snafu(display(
1160 "Timestamp column mismatch, expected column_name: {}, expected column_id: {}, actual column_name: {}, actual column_id: {}",
1161 expected_column_name,
1162 expected_column_id,
1163 actual_column_name,
1164 actual_column_id,
1165 ))]
1166 TimestampMismatch {
1167 expected_column_name: String,
1168 expected_column_id: u32,
1169 actual_column_name: String,
1170 actual_column_id: u32,
1171 },
1172
1173 #[snafu(display(
1174 "Partition expression of region {} is incompatible with the altered column '{}': the persisted bound can no longer be compared with the new column type {}",
1175 region_id,
1176 column,
1177 target_type
1178 ))]
1179 PartitionExprIncompatible {
1180 region_id: RegionId,
1181 column: String,
1182 target_type: String,
1183 #[snafu(implicit)]
1184 location: Location,
1185 },
1186
1187 #[cfg(feature = "enterprise")]
1188 #[snafu(display("Too large duration"))]
1189 TooLargeDuration {
1190 #[snafu(source)]
1191 error: prost_types::DurationError,
1192 #[snafu(implicit)]
1193 location: Location,
1194 },
1195
1196 #[cfg(feature = "enterprise")]
1197 #[snafu(display("Negative duration"))]
1198 NegativeDuration {
1199 #[snafu(source)]
1200 error: prost_types::DurationError,
1201 #[snafu(implicit)]
1202 location: Location,
1203 },
1204
1205 #[cfg(feature = "enterprise")]
1206 #[snafu(display("Missing interval field"))]
1207 MissingInterval {
1208 #[snafu(implicit)]
1209 location: Location,
1210 },
1211}
1212
1213pub type Result<T> = std::result::Result<T, Error>;
1214
1215impl ErrorExt for Error {
1216 fn status_code(&self) -> StatusCode {
1217 use Error::*;
1218 match self {
1219 Error::PartialRegion { response } => response
1220 .first_failure()
1221 .map_or(StatusCode::Internal, |(_, error)| error.status_code()),
1222 Error::BatchRequest { source } => source.status_code(),
1223 IllegalServerState { .. }
1224 | EtcdTxnOpResponse { .. }
1225 | EtcdFailed { .. }
1226 | EtcdTxnFailed { .. }
1227 | ConnectEtcd { .. }
1228 | MoveValues { .. }
1229 | TombstoneTargetAlreadyExists { .. }
1230 | GetCache { .. }
1231 | GetLatestCacheRetryExceeded { .. }
1232 | SerializeToJson { .. }
1233 | DeserializeFromJson { .. }
1234 | ElectionNoLeader { .. }
1235 | ElectionLeaderLeaseExpired { .. }
1236 | ElectionLeaderLeaseChanged { .. } => StatusCode::Internal,
1237
1238 NoLeader { .. } => StatusCode::TableUnavailable,
1239 ValueNotExist { .. }
1240 | ProcedurePoisonConflict { .. }
1241 | ProcedureStateReceiverNotFound { .. }
1242 | MissingColumnIds { .. }
1243 | MissingColumnInColumnMetadata { .. }
1244 | MismatchColumnId { .. }
1245 | ColumnMetadataConflicts { .. }
1246 | ColumnNotFound { .. }
1247 | ColumnIdMismatch { .. }
1248 | TimestampMismatch { .. } => StatusCode::Unexpected,
1249
1250 Unsupported { .. } | ReadOnlyKvBackend { .. } => StatusCode::Unsupported,
1251 WriteObject { .. } | ReadObject { .. } => StatusCode::StorageUnavailable,
1252
1253 SerdeJson { .. }
1254 | ParseOption { .. }
1255 | InvalidProtoMsg { .. }
1256 | InvalidMetadata { .. }
1257 | Unexpected { .. }
1258 | TableInfoNotFound { .. }
1259 | NextSequence { .. }
1260 | UnexpectedSequenceValue { .. }
1261 | InvalidHeartbeatResponse { .. }
1262 | EncodeJson { .. }
1263 | DecodeJson { .. }
1264 | PayloadNotExist { .. }
1265 | ConvertRawKey { .. }
1266 | DecodeProto { .. }
1267 | DecodePackedFileRefs { .. }
1268 | InvalidPackedFileRefs { .. }
1269 | BuildTableMeta { .. }
1270 | TableRouteNotFound { .. }
1271 | TableRepartNotFound { .. }
1272 | RegionOperatingRace { .. }
1273 | SerializeWalOptions { .. }
1274 | BuildKafkaClient { .. }
1275 | BuildKafkaCtrlClient { .. }
1276 | KafkaPartitionClient { .. }
1277 | ProduceRecord { .. }
1278 | CreateKafkaWalTopic { .. }
1279 | EmptyTopicPool { .. }
1280 | UnexpectedLogicalRouteTable { .. }
1281 | ProcedureOutput { .. }
1282 | FromUtf8 { .. }
1283 | MetadataCorruption { .. }
1284 | KafkaGetOffset { .. }
1285 | ReadFlexbuffers { .. }
1286 | SerializeFlexbuffers { .. }
1287 | DeserializeFlexbuffers { .. }
1288 | ConvertTimeRanges { .. } => StatusCode::Unexpected,
1289
1290 GetKvCache { .. } | CacheNotGet { .. } => StatusCode::Internal,
1291
1292 SchemaAlreadyExists { .. } => StatusCode::DatabaseAlreadyExists,
1293
1294 ProcedureNotFound { .. }
1295 | InvalidViewInfo { .. }
1296 | InvalidFlowOption { .. }
1297 | PrimaryKeyNotFound { .. }
1298 | EmptyKey { .. }
1299 | AlterLogicalTablesInvalidArguments { .. }
1300 | CreateLogicalTablesInvalidArguments { .. }
1301 | MismatchPrefix { .. }
1302 | TlsConfig { .. }
1303 | InvalidSetDatabaseOption { .. }
1304 | InvalidUnsetDatabaseOption { .. }
1305 | InvalidTopicNamePrefix { .. }
1306 | InvalidFileExtension { .. }
1307 | InvalidFileName { .. }
1308 | InvalidFlowRequestBody { .. }
1309 | InvalidFilePath { .. }
1310 | ConflictingSchemaOptions { .. }
1311 | PartitionExprIncompatible { .. } => StatusCode::InvalidArguments,
1312
1313 #[cfg(feature = "enterprise")]
1314 MissingInterval { .. } | NegativeDuration { .. } | TooLargeDuration { .. } => {
1315 StatusCode::InvalidArguments
1316 }
1317
1318 FlowNotFound { .. } => StatusCode::FlowNotFound,
1319 FlowRouteNotFound { .. } => StatusCode::Unexpected,
1320 FlowAlreadyExists { .. } => StatusCode::FlowAlreadyExists,
1321
1322 ViewNotFound { .. } | TableNotFound { .. } | RegionNotFound { .. } => {
1323 StatusCode::TableNotFound
1324 }
1325 ViewAlreadyExists { .. }
1326 | TableAlreadyExists { .. }
1327 | TableNameTombstoneConflict { .. } => StatusCode::TableAlreadyExists,
1328
1329 SubmitProcedure { source, .. }
1330 | QueryProcedure { source, .. }
1331 | WaitProcedure { source, .. }
1332 | StartProcedureManager { source, .. }
1333 | StopProcedureManager { source, .. } => source.status_code(),
1334 RegisterProcedureLoader { source, .. } => source.status_code(),
1335 External { source, .. } => source.status_code(),
1336 ResponseExceededSizeLimit { source, .. } => source.status_code(),
1337 OperateDatanode { source, .. } => source.status_code(),
1338 Table { source, .. } => source.status_code(),
1339 RetryLater { source, .. } => source.status_code(),
1340 AbortProcedure { source, .. } => source.status_code(),
1341 ConvertAlterTableRequest { source, .. } => source.status_code(),
1342 PutPoison { source, .. } => source.status_code(),
1343 ConvertColumnDef { source, .. } => source.status_code(),
1344 ProcedureStateReceiver { source, .. } => source.status_code(),
1345 RegisterRepartitionProcedureLoader { source, .. } => source.status_code(),
1346 CreateRepartitionProcedure { source, .. } => source.status_code(),
1347 PersistRepartitionGcRequirement { source, .. } => source.status_code(),
1348
1349 ParseProcedureId { .. }
1350 | InvalidNumTopics { .. }
1351 | SchemaNotFound { .. }
1352 | CatalogNotFound { .. }
1353 | InvalidNodeInfoKey { .. }
1354 | InvalidStatKey { .. }
1355 | ParseNum { .. }
1356 | InvalidRole { .. }
1357 | EmptyDdlTasks { .. } => StatusCode::InvalidArguments,
1358
1359 DdlTaskTimeout { .. } => StatusCode::Cancelled,
1362 LoadTlsCertificate { .. } => StatusCode::Internal,
1363
1364 #[cfg(feature = "pg_kvbackend")]
1365 PostgresExecution { .. }
1366 | CreatePostgresPool { .. }
1367 | GetPostgresConnection { .. }
1368 | GetPostgresClient { .. }
1369 | PostgresTransaction { .. }
1370 | PostgresTlsConfig { .. }
1371 | InvalidTlsConfig { .. } => StatusCode::Internal,
1372 #[cfg(feature = "mysql_kvbackend")]
1373 MySqlExecution { .. }
1374 | CreateMySqlPool { .. }
1375 | DecodeSqlValue { .. }
1376 | AcquireMySqlClient { .. }
1377 | MySqlTransaction { .. } => StatusCode::Internal,
1378 #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
1379 RdsTransactionRetryFailed { .. } | SqlExecutionTimeout { .. } => StatusCode::Internal,
1380 DatanodeTableInfoNotFound { .. } => StatusCode::Internal,
1381 }
1382 }
1383
1384 fn as_any(&self) -> &dyn std::any::Any {
1385 self
1386 }
1387
1388 fn retry_hint(&self) -> RetryHint {
1389 use Error::*;
1390
1391 match self {
1392 Error::PartialRegion { .. } => RetryHint::Retryable,
1393 Error::BatchRequest { source } => source.retry_hint(),
1394 RetryLater { source, .. } => match source.retry_hint() {
1395 RetryHint::RetryAfterRouteRefresh => RetryHint::RetryAfterRouteRefresh,
1396 RetryHint::Retryable | RetryHint::NonRetryable => RetryHint::Retryable,
1397 },
1398 GetLatestCacheRetryExceeded { .. }
1399 | NoLeader { .. }
1400 | ElectionNoLeader { .. }
1401 | ElectionLeaderLeaseExpired { .. }
1402 | ElectionLeaderLeaseChanged { .. } => RetryHint::Retryable,
1403 ConnectEtcd { error, .. } | EtcdFailed { error, .. } | EtcdTxnFailed { error, .. } => {
1404 retry_hint_from_etcd_error(error)
1405 }
1406 WriteObject { error, .. } | ReadObject { error, .. } => {
1407 retry_hint_from_opendal_error(error)
1408 }
1409 BuildKafkaClient { error, .. }
1410 | BuildKafkaCtrlClient { error, .. }
1411 | KafkaPartitionClient { error, .. }
1412 | KafkaGetOffset { error, .. }
1413 | ProduceRecord { error, .. }
1414 | CreateKafkaWalTopic { error, .. } => rskafka_client_error_to_retry_hint(error),
1415 SubmitProcedure { source, .. }
1416 | QueryProcedure { source, .. }
1417 | WaitProcedure { source, .. }
1418 | StartProcedureManager { source, .. }
1419 | StopProcedureManager { source, .. }
1420 | RegisterProcedureLoader { source, .. }
1421 | PutPoison { source, .. } => source.retry_hint(),
1422 External { source, .. }
1423 | ResponseExceededSizeLimit { source, .. }
1424 | OperateDatanode { source, .. }
1425 | RegisterRepartitionProcedureLoader { source, .. }
1426 | CreateRepartitionProcedure { source, .. }
1427 | PersistRepartitionGcRequirement { source, .. } => source.retry_hint(),
1428 AbortProcedure { .. } | ProcedureStateReceiver { .. } => RetryHint::NonRetryable,
1430 Table { source, .. } => source.retry_hint(),
1431 ConvertAlterTableRequest { source, .. } => source.retry_hint(),
1432 ConvertColumnDef { source, .. } => source.retry_hint(),
1433 GetCache { source, .. } => source.retry_hint(),
1434 #[cfg(feature = "pg_kvbackend")]
1435 PostgresExecution { error, .. } | PostgresTransaction { error, .. } => {
1436 retry_hint_from_postgres_error(error)
1437 }
1438 #[cfg(feature = "pg_kvbackend")]
1439 GetPostgresClient { error, .. } => retry_hint_from_postgres_pool_error(error),
1440 #[cfg(feature = "mysql_kvbackend")]
1441 MySqlExecution { error, .. }
1442 | CreateMySqlPool { error, .. }
1443 | AcquireMySqlClient { error, .. }
1444 | MySqlTransaction { error, .. } => retry_hint_from_sqlx_error(error),
1445 #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
1446 RdsTransactionRetryFailed { .. } | SqlExecutionTimeout { .. } => RetryHint::Retryable,
1447 _ => RetryHint::NonRetryable,
1448 }
1449 }
1450}
1451
1452impl Error {
1453 #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
1454 pub fn is_serialization_error(&self) -> bool {
1456 match self {
1457 #[cfg(feature = "pg_kvbackend")]
1458 Error::PostgresExecution { error, .. } | Error::PostgresTransaction { error, .. } => {
1459 retry_hint::is_postgres_serialization_error(error)
1460 }
1461 #[cfg(feature = "mysql_kvbackend")]
1462 Error::MySqlExecution { error, .. } | Error::MySqlTransaction { error, .. } => {
1463 retry_hint::is_mysql_serialization_error(error)
1464 }
1465 _ => false,
1466 }
1467 }
1468
1469 pub fn retry_later<E: ErrorExt + Send + Sync + 'static>(err: E) -> Error {
1471 Error::RetryLater {
1472 source: BoxedError::new(err),
1473 clean_poisons: false,
1474 }
1475 }
1476
1477 pub fn is_retry_later(&self) -> bool {
1479 matches!(
1480 self,
1481 Error::RetryLater { .. } | Error::GetLatestCacheRetryExceeded { .. }
1482 )
1483 }
1484
1485 pub fn need_clean_poisons(&self) -> bool {
1487 matches!(
1488 self,
1489 Error::AbortProcedure { clean_poisons, .. } if *clean_poisons
1490 ) || matches!(
1491 self,
1492 Error::RetryLater { clean_poisons, .. } if *clean_poisons
1493 )
1494 }
1495
1496 pub fn is_exceeded_size_limit(&self) -> bool {
1498 match self {
1499 Error::EtcdFailed {
1500 error: etcd_client::Error::GRpcStatus(status),
1501 ..
1502 } => status.code() == tonic::Code::OutOfRange,
1503 Error::ResponseExceededSizeLimit { .. } => true,
1504 _ => false,
1505 }
1506 }
1507}
1508
1509#[cfg(test)]
1510mod retry_hint_tests {
1511 use std::sync::Arc;
1512
1513 use common_error::mock::MockError;
1514 use snafu::ResultExt;
1515
1516 use super::*;
1517
1518 #[test]
1519 fn test_retry_later_hint_is_retryable() {
1520 let err = Error::retry_later(MockError::new(StatusCode::Internal));
1521
1522 assert_eq!(err.retry_hint(), RetryHint::Retryable);
1523 }
1524
1525 #[test]
1526 fn test_latest_cache_retry_exceeded_hint_is_retryable() {
1527 let err = GetLatestCacheRetryExceededSnafu { attempts: 3_usize }.build();
1528
1529 assert_eq!(err.retry_hint(), RetryHint::Retryable);
1530 }
1531
1532 #[test]
1533 fn test_leader_retry_hints() {
1534 for (error, expected) in [
1535 (
1536 NoLeaderSnafu { table_id: 1_u32 }.build(),
1537 RetryHint::Retryable,
1538 ),
1539 (ElectionNoLeaderSnafu.build(), RetryHint::Retryable),
1540 (
1541 ElectionLeaderLeaseExpiredSnafu.build(),
1542 RetryHint::Retryable,
1543 ),
1544 (
1545 ElectionLeaderLeaseChangedSnafu.build(),
1546 RetryHint::Retryable,
1547 ),
1548 ] {
1549 assert_eq!(error.retry_hint(), expected, "{error:?}");
1550 }
1551 }
1552
1553 #[test]
1554 fn test_procedure_error_retry_hints() {
1555 let source = common_procedure::Error::retry_later(MockError::new(StatusCode::Internal));
1556 let err = Err::<(), _>(source).context(PutPoisonSnafu).unwrap_err();
1557 assert_eq!(err.retry_hint(), RetryHint::Retryable);
1558
1559 let source = common_procedure::Error::retry_later(MockError::new(StatusCode::Internal));
1560 let err = Err::<(), _>(source)
1561 .context(ProcedureStateReceiverSnafu {
1562 procedure_id: ProcedureId::random(),
1563 })
1564 .unwrap_err();
1565 assert_eq!(err.retry_hint(), RetryHint::NonRetryable);
1566 }
1567
1568 #[test]
1569 fn test_get_cache_forwards_retry_hint() {
1570 let source = Arc::new(Error::retry_later(MockError::new(StatusCode::Internal)));
1571 let err = Error::GetCache { source };
1572
1573 assert_eq!(err.retry_hint(), RetryHint::Retryable);
1574 }
1575
1576 #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
1577 #[test]
1578 fn test_sql_execution_timeout_hint_is_retryable() {
1579 let err = SqlExecutionTimeoutSnafu {
1580 sql: "SELECT 1".to_string(),
1581 duration: std::time::Duration::from_secs(1),
1582 }
1583 .build();
1584
1585 assert_eq!(err.retry_hint(), RetryHint::Retryable);
1586 }
1587
1588 #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
1589 #[test]
1590 fn test_rds_transaction_retry_failed_hint_is_retryable() {
1591 let err = RdsTransactionRetryFailedSnafu.build();
1592
1593 assert_eq!(err.retry_hint(), RetryHint::Retryable);
1594 }
1595
1596 #[test]
1597 fn test_default_hint_is_non_retryable() {
1598 let err = UnexpectedSnafu {
1599 err_msg: "mock error",
1600 }
1601 .build();
1602
1603 assert_eq!(err.retry_hint(), RetryHint::NonRetryable);
1604 }
1605}