1use std::str::Utf8Error;
16use std::sync::Arc;
17
18use common_error::ext::{BoxedError, ErrorExt, RetryHint};
19use common_error::status_code::StatusCode;
20use common_macro::stack_trace_debug;
21use common_procedure::ProcedureId;
22use common_wal::kafka::rskafka_client_error_to_retry_hint;
23use object_store::error::retry_hint_from_opendal_error;
24use serde_json::error::Error as JsonError;
25use snafu::{Location, Snafu};
26use store_api::storage::RegionId;
27use table::metadata::TableId;
28
29use crate::DatanodeId;
30use crate::peer::Peer;
31
32mod retry_hint;
33
34pub use retry_hint::retry_hint_from_etcd_error;
35#[cfg(feature = "mysql_kvbackend")]
36pub use retry_hint::retry_hint_from_sqlx_error;
37#[cfg(feature = "pg_kvbackend")]
38pub use retry_hint::{retry_hint_from_postgres_error, retry_hint_from_postgres_pool_error};
39
40#[derive(Snafu)]
41#[snafu(visibility(pub))]
42#[stack_trace_debug]
43pub enum Error {
44 #[snafu(display(
45 "Region request has {} failed entries; {}",
46 response.failures.len(),
47 response.first_failure().map(|(index, error)| format!("entry {index}: {}", error.output_msg())).unwrap_or_default()
48 ))]
49 PartialRegion {
50 response: Box<api::region::BatchRegionResponse<Error>>,
51 },
52
53 #[snafu(display("Empty key is not allowed"))]
54 EmptyKey {
55 #[snafu(implicit)]
56 location: Location,
57 },
58
59 #[snafu(display(
60 "Another procedure is operating the region: {} on peer: {}",
61 region_id,
62 peer_id
63 ))]
64 RegionOperatingRace {
65 #[snafu(implicit)]
66 location: Location,
67 peer_id: DatanodeId,
68 region_id: RegionId,
69 },
70
71 #[snafu(display("Failed to connect to Etcd"))]
72 ConnectEtcd {
73 #[snafu(source)]
74 error: etcd_client::Error,
75 #[snafu(implicit)]
76 location: Location,
77 },
78
79 #[snafu(display("Failed to execute via Etcd"))]
80 EtcdFailed {
81 #[snafu(source)]
82 error: etcd_client::Error,
83 #[snafu(implicit)]
84 location: Location,
85 },
86
87 #[snafu(display("Failed to execute {} txn operations via Etcd", max_operations))]
88 EtcdTxnFailed {
89 max_operations: usize,
90 #[snafu(source)]
91 error: etcd_client::Error,
92 #[snafu(implicit)]
93 location: Location,
94 },
95
96 #[snafu(display("Failed to get sequence: {}", err_msg))]
97 NextSequence {
98 err_msg: String,
99 #[snafu(implicit)]
100 location: Location,
101 },
102
103 #[snafu(display("Unexpected sequence value: {}", err_msg))]
104 UnexpectedSequenceValue {
105 err_msg: String,
106 #[snafu(implicit)]
107 location: Location,
108 },
109
110 #[snafu(display("Table info not found: {}", table))]
111 TableInfoNotFound {
112 table: String,
113 #[snafu(implicit)]
114 location: Location,
115 },
116
117 #[snafu(display("Failed to register procedure loader, type name: {}", type_name))]
118 RegisterProcedureLoader {
119 type_name: String,
120 #[snafu(implicit)]
121 location: Location,
122 source: common_procedure::error::Error,
123 },
124
125 #[snafu(display("Failed to register repartition procedure loader"))]
126 RegisterRepartitionProcedureLoader {
127 #[snafu(implicit)]
128 location: Location,
129 source: BoxedError,
130 },
131
132 #[snafu(display("Failed to create repartition procedure"))]
133 CreateRepartitionProcedure {
134 source: BoxedError,
135 #[snafu(implicit)]
136 location: Location,
137 },
138
139 #[snafu(display("Failed to persist the repartition GC requirement"))]
140 PersistRepartitionGcRequirement {
141 source: BoxedError,
142 #[snafu(implicit)]
143 location: Location,
144 },
145
146 #[snafu(display("Failed to submit procedure"))]
147 SubmitProcedure {
148 #[snafu(implicit)]
149 location: Location,
150 source: common_procedure::Error,
151 },
152
153 #[snafu(display("Failed to query procedure"))]
154 QueryProcedure {
155 #[snafu(implicit)]
156 location: Location,
157 source: common_procedure::Error,
158 },
159
160 #[snafu(display("Procedure not found: {pid}"))]
161 ProcedureNotFound {
162 #[snafu(implicit)]
163 location: Location,
164 pid: String,
165 },
166
167 #[snafu(display("Failed to parse procedure id: {key}"))]
168 ParseProcedureId {
169 #[snafu(implicit)]
170 location: Location,
171 key: String,
172 #[snafu(source)]
173 error: common_procedure::ParseIdError,
174 },
175
176 #[snafu(display("Unsupported operation {}", operation))]
177 Unsupported {
178 operation: String,
179 #[snafu(implicit)]
180 location: Location,
181 },
182
183 #[snafu(display("Trying to write to a read-only kv backend: {}", name))]
184 ReadOnlyKvBackend {
185 name: String,
186 #[snafu(implicit)]
187 location: Location,
188 },
189
190 #[snafu(display("Failed to get procedure state receiver, procedure id: {procedure_id}"))]
191 ProcedureStateReceiver {
192 procedure_id: ProcedureId,
193 #[snafu(implicit)]
194 location: Location,
195 source: common_procedure::Error,
196 },
197
198 #[snafu(display("Procedure state receiver not found: {procedure_id}"))]
199 ProcedureStateReceiverNotFound {
200 procedure_id: ProcedureId,
201 #[snafu(implicit)]
202 location: Location,
203 },
204
205 #[snafu(display("Failed to wait procedure done"))]
206 WaitProcedure {
207 #[snafu(implicit)]
208 location: Location,
209 source: common_procedure::Error,
210 },
211
212 #[snafu(display(
213 "DDL request timed out after {timeout:?}; a submitted procedure may still be running"
214 ))]
215 DdlTaskTimeout {
216 timeout: std::time::Duration,
217 #[snafu(implicit)]
218 location: Location,
219 },
220
221 #[snafu(display("Failed to start procedure manager"))]
222 StartProcedureManager {
223 #[snafu(implicit)]
224 location: Location,
225 source: common_procedure::Error,
226 },
227
228 #[snafu(display("Failed to stop procedure manager"))]
229 StopProcedureManager {
230 #[snafu(implicit)]
231 location: Location,
232 source: common_procedure::Error,
233 },
234
235 #[snafu(display(
236 "Failed to get procedure output, procedure id: {procedure_id}, error: {err_msg}"
237 ))]
238 ProcedureOutput {
239 procedure_id: String,
240 err_msg: String,
241 #[snafu(implicit)]
242 location: Location,
243 },
244
245 #[snafu(display("Primary key '{key}' not found when creating region request"))]
246 PrimaryKeyNotFound {
247 key: String,
248 #[snafu(implicit)]
249 location: Location,
250 },
251
252 #[snafu(display("Failed to build table meta for table: {}", table_name))]
253 BuildTableMeta {
254 table_name: String,
255 #[snafu(source)]
256 error: table::metadata::TableMetaBuilderError,
257 #[snafu(implicit)]
258 location: Location,
259 },
260
261 #[snafu(display("Table occurs error"))]
262 Table {
263 #[snafu(implicit)]
264 location: Location,
265 source: table::error::Error,
266 },
267
268 #[snafu(display("Failed to find table route for table id {}", table_id))]
269 TableRouteNotFound {
270 table_id: TableId,
271 #[snafu(implicit)]
272 location: Location,
273 },
274
275 #[snafu(display("Failed to find table repartition metadata for table id {}", table_id))]
276 TableRepartNotFound {
277 table_id: TableId,
278 #[snafu(implicit)]
279 location: Location,
280 },
281
282 #[snafu(display("Failed to decode protobuf"))]
283 DecodeProto {
284 #[snafu(implicit)]
285 location: Location,
286 #[snafu(source)]
287 error: prost::DecodeError,
288 },
289
290 #[snafu(display("Failed to decode packed file references"))]
291 DecodePackedFileRefs {
292 #[snafu(implicit)]
293 location: Location,
294 #[snafu(source)]
295 error: base64::DecodeError,
296 },
297
298 #[snafu(display("Invalid packed file references framing"))]
299 InvalidPackedFileRefs {
300 #[snafu(implicit)]
301 location: Location,
302 },
303
304 #[snafu(display("Failed to encode object into json"))]
305 EncodeJson {
306 #[snafu(implicit)]
307 location: Location,
308 #[snafu(source)]
309 error: JsonError,
310 },
311
312 #[snafu(display("Failed to decode object from json"))]
313 DecodeJson {
314 #[snafu(implicit)]
315 location: Location,
316 #[snafu(source)]
317 error: JsonError,
318 },
319
320 #[snafu(display("Failed to serialize to json: {}", input))]
321 SerializeToJson {
322 input: String,
323 #[snafu(source)]
324 error: serde_json::error::Error,
325 #[snafu(implicit)]
326 location: Location,
327 },
328
329 #[snafu(display("Failed to deserialize from json: {}", input))]
330 DeserializeFromJson {
331 input: String,
332 #[snafu(source)]
333 error: serde_json::error::Error,
334 #[snafu(implicit)]
335 location: Location,
336 },
337
338 #[snafu(display("Payload not exist"))]
339 PayloadNotExist {
340 #[snafu(implicit)]
341 location: Location,
342 },
343
344 #[snafu(display("Failed to serde json"))]
345 SerdeJson {
346 #[snafu(source)]
347 error: serde_json::error::Error,
348 #[snafu(implicit)]
349 location: Location,
350 },
351
352 #[snafu(display("Failed to parse value {} into key {}", value, key))]
353 ParseOption {
354 key: String,
355 value: String,
356 #[snafu(implicit)]
357 location: Location,
358 },
359
360 #[snafu(display(
361 "Conflicting schema options: {}={} and {}={}",
362 first_key,
363 first_value,
364 second_key,
365 second_value
366 ))]
367 ConflictingSchemaOptions {
368 first_key: String,
369 first_value: String,
370 second_key: String,
371 second_value: String,
372 #[snafu(implicit)]
373 location: Location,
374 },
375
376 #[snafu(display("Illegal state from server, code: {}, error: {}", code, err_msg))]
377 IllegalServerState {
378 code: i32,
379 err_msg: String,
380 #[snafu(implicit)]
381 location: Location,
382 },
383
384 #[snafu(display("Failed to convert alter table request"))]
385 ConvertAlterTableRequest {
386 source: common_grpc_expr::error::Error,
387 #[snafu(implicit)]
388 location: Location,
389 },
390
391 #[snafu(display("Invalid protobuf message: {err_msg}"))]
392 InvalidProtoMsg {
393 err_msg: String,
394 #[snafu(implicit)]
395 location: Location,
396 },
397
398 #[snafu(display("Unexpected: {err_msg}"))]
399 Unexpected {
400 err_msg: String,
401 #[snafu(implicit)]
402 location: Location,
403 },
404
405 #[snafu(display("Metasrv election has no leader at this moment"))]
406 ElectionNoLeader {
407 #[snafu(implicit)]
408 location: Location,
409 },
410
411 #[snafu(display("Metasrv election leader lease expired"))]
412 ElectionLeaderLeaseExpired {
413 #[snafu(implicit)]
414 location: Location,
415 },
416
417 #[snafu(display("Metasrv election leader lease changed during election"))]
418 ElectionLeaderLeaseChanged {
419 #[snafu(implicit)]
420 location: Location,
421 },
422
423 #[snafu(display("Table already exists, table: {}", table_name))]
424 TableAlreadyExists {
425 table_name: String,
426 #[snafu(implicit)]
427 location: Location,
428 },
429
430 #[snafu(display(
431 "Cannot drop table '{}': an older tombstone already uses the same full name",
432 table_name
433 ))]
434 TableNameTombstoneConflict {
436 table_name: String,
437 existing_table_id: TableId,
438 dropping_table_id: TableId,
439 #[snafu(implicit)]
440 location: Location,
441 },
442
443 #[snafu(display("View already exists, view: {}", view_name))]
444 ViewAlreadyExists {
445 view_name: String,
446 #[snafu(implicit)]
447 location: Location,
448 },
449
450 #[snafu(display("Flow already exists: {}", flow_name))]
451 FlowAlreadyExists {
452 flow_name: String,
453 #[snafu(implicit)]
454 location: Location,
455 },
456
457 #[snafu(display("Schema already exists, catalog:{}, schema: {}", catalog, schema))]
458 SchemaAlreadyExists {
459 catalog: String,
460 schema: String,
461 #[snafu(implicit)]
462 location: Location,
463 },
464
465 #[snafu(display("Failed to convert raw key to str"))]
466 ConvertRawKey {
467 #[snafu(implicit)]
468 location: Location,
469 #[snafu(source)]
470 error: Utf8Error,
471 },
472
473 #[snafu(display("Table not found: '{}'", table_name))]
474 TableNotFound {
475 table_name: String,
476 #[snafu(implicit)]
477 location: Location,
478 },
479
480 #[snafu(display("Region not found: {}", region_id))]
481 RegionNotFound {
482 region_id: RegionId,
483 #[snafu(implicit)]
484 location: Location,
485 },
486
487 #[snafu(display("View not found: '{}'", view_name))]
488 ViewNotFound {
489 view_name: String,
490 #[snafu(implicit)]
491 location: Location,
492 },
493
494 #[snafu(display("Flow not found: '{}'", flow_name))]
495 FlowNotFound {
496 flow_name: String,
497 #[snafu(implicit)]
498 location: Location,
499 },
500
501 #[snafu(display("Flow route not found: '{}'", flow_name))]
502 FlowRouteNotFound {
503 flow_name: String,
504 #[snafu(implicit)]
505 location: Location,
506 },
507
508 #[snafu(display("Schema nod found, schema: {}", table_schema))]
509 SchemaNotFound {
510 table_schema: String,
511 #[snafu(implicit)]
512 location: Location,
513 },
514
515 #[snafu(display("Catalog not found, catalog: {}", catalog))]
516 CatalogNotFound {
517 catalog: String,
518 #[snafu(implicit)]
519 location: Location,
520 },
521
522 #[snafu(display("Invalid metadata, err: {}", err_msg))]
523 InvalidMetadata {
524 err_msg: String,
525 #[snafu(implicit)]
526 location: Location,
527 },
528
529 #[snafu(display("Invalid view info, err: {}", err_msg))]
530 InvalidViewInfo {
531 err_msg: String,
532 #[snafu(implicit)]
533 location: Location,
534 },
535
536 #[snafu(display("Invalid SQL, error: {err_msg}"))]
537 InvalidFlowOption {
538 err_msg: String,
539 #[snafu(implicit)]
540 location: Location,
541 },
542
543 #[snafu(display("Invalid flow request body: {:?}", body))]
544 InvalidFlowRequestBody {
545 body: Box<Option<api::v1::flow::flow_request::Body>>,
546 #[snafu(implicit)]
547 location: Location,
548 },
549
550 #[snafu(display("Failed to get kv cache, err: {}", err_msg))]
551 GetKvCache { err_msg: String },
552
553 #[snafu(display("Get null from cache, key: {}", key))]
554 CacheNotGet {
555 key: String,
556 #[snafu(implicit)]
557 location: Location,
558 },
559
560 #[snafu(display("Etcd txn error: {err_msg}"))]
561 EtcdTxnOpResponse {
562 err_msg: String,
563 #[snafu(implicit)]
564 location: Location,
565 },
566
567 #[snafu(display("External error"))]
568 External {
569 #[snafu(implicit)]
570 location: Location,
571 source: BoxedError,
572 },
573
574 #[snafu(display("The response exceeded size limit"))]
575 ResponseExceededSizeLimit {
576 #[snafu(implicit)]
577 location: Location,
578 source: BoxedError,
579 },
580
581 #[snafu(display("Invalid heartbeat response"))]
582 InvalidHeartbeatResponse {
583 #[snafu(implicit)]
584 location: Location,
585 },
586
587 #[snafu(display("Failed to operate on datanode: {}", peer))]
588 OperateDatanode {
589 #[snafu(implicit)]
590 location: Location,
591 peer: Peer,
592 source: BoxedError,
593 },
594
595 #[snafu(display("Retry later"))]
596 RetryLater {
597 source: BoxedError,
598 clean_poisons: bool,
599 },
600
601 #[snafu(display("Abort procedure"))]
602 AbortProcedure {
603 #[snafu(implicit)]
604 location: Location,
605 source: BoxedError,
606 clean_poisons: bool,
607 },
608
609 #[snafu(display("Failed to serialize WAL options for region: {region_id}"))]
610 SerializeWalOptions {
611 region_id: RegionId,
612 #[snafu(source)]
613 error: serde_json::Error,
614 #[snafu(implicit)]
615 location: Location,
616 },
617
618 #[snafu(display("Invalid number of topics {}", num_topics))]
619 InvalidNumTopics {
620 num_topics: usize,
621 #[snafu(implicit)]
622 location: Location,
623 },
624
625 #[snafu(display(
626 "Failed to build a Kafka client, broker endpoints: {:?}",
627 broker_endpoints
628 ))]
629 BuildKafkaClient {
630 broker_endpoints: Vec<String>,
631 #[snafu(implicit)]
632 location: Location,
633 #[snafu(source)]
634 error: rskafka::client::error::Error,
635 },
636
637 #[snafu(display("Failed to create TLS Config"))]
638 TlsConfig {
639 #[snafu(implicit)]
640 location: Location,
641 source: common_wal::error::Error,
642 },
643
644 #[snafu(display("Failed to build a Kafka controller client"))]
645 BuildKafkaCtrlClient {
646 #[snafu(implicit)]
647 location: Location,
648 #[snafu(source)]
649 error: rskafka::client::error::Error,
650 },
651
652 #[snafu(display(
653 "Failed to get a Kafka partition client, topic: {}, partition: {}",
654 topic,
655 partition
656 ))]
657 KafkaPartitionClient {
658 topic: String,
659 partition: i32,
660 #[snafu(implicit)]
661 location: Location,
662 #[snafu(source)]
663 error: rskafka::client::error::Error,
664 },
665
666 #[snafu(display(
667 "Failed to get offset from Kafka, topic: {}, partition: {}",
668 topic,
669 partition
670 ))]
671 KafkaGetOffset {
672 topic: String,
673 partition: i32,
674 #[snafu(implicit)]
675 location: Location,
676 #[snafu(source)]
677 error: rskafka::client::error::Error,
678 },
679
680 #[snafu(display("Failed to produce records to Kafka, topic: {}", topic))]
681 ProduceRecord {
682 topic: String,
683 #[snafu(implicit)]
684 location: Location,
685 #[snafu(source)]
686 error: rskafka::client::error::Error,
687 },
688
689 #[snafu(display("Failed to create a Kafka wal topic"))]
690 CreateKafkaWalTopic {
691 #[snafu(implicit)]
692 location: Location,
693 #[snafu(source)]
694 error: rskafka::client::error::Error,
695 },
696
697 #[snafu(display("The topic pool is empty"))]
698 EmptyTopicPool {
699 #[snafu(implicit)]
700 location: Location,
701 },
702
703 #[snafu(display("Unexpected table route type: {}", err_msg))]
704 UnexpectedLogicalRouteTable {
705 #[snafu(implicit)]
706 location: Location,
707 err_msg: String,
708 },
709
710 #[snafu(display("The tasks of {} cannot be empty", name))]
711 EmptyDdlTasks {
712 name: String,
713 #[snafu(implicit)]
714 location: Location,
715 },
716
717 #[snafu(display("Metadata corruption: {}", err_msg))]
718 MetadataCorruption {
719 err_msg: String,
720 #[snafu(implicit)]
721 location: Location,
722 },
723
724 #[snafu(display("Alter logical tables invalid arguments: {}", err_msg))]
725 AlterLogicalTablesInvalidArguments {
726 err_msg: String,
727 #[snafu(implicit)]
728 location: Location,
729 },
730
731 #[snafu(display("Create logical tables invalid arguments: {}", err_msg))]
732 CreateLogicalTablesInvalidArguments {
733 err_msg: String,
734 #[snafu(implicit)]
735 location: Location,
736 },
737
738 #[snafu(display("Invalid node info key: {}", key))]
739 InvalidNodeInfoKey {
740 key: String,
741 #[snafu(implicit)]
742 location: Location,
743 },
744
745 #[snafu(display("Invalid node stat key: {}", key))]
746 InvalidStatKey {
747 key: String,
748 #[snafu(implicit)]
749 location: Location,
750 },
751
752 #[snafu(display("Failed to parse number: {}", err_msg))]
753 ParseNum {
754 err_msg: String,
755 #[snafu(source)]
756 error: std::num::ParseIntError,
757 #[snafu(implicit)]
758 location: Location,
759 },
760
761 #[snafu(display("Invalid role: {}", role))]
762 InvalidRole {
763 role: i32,
764 #[snafu(implicit)]
765 location: Location,
766 },
767
768 #[snafu(display("Invalid set database option, key: {}, value: {}", key, value))]
769 InvalidSetDatabaseOption {
770 key: String,
771 value: String,
772 #[snafu(implicit)]
773 location: Location,
774 },
775
776 #[snafu(display("Invalid unset database option, key: {}", key))]
777 InvalidUnsetDatabaseOption {
778 key: String,
779 #[snafu(implicit)]
780 location: Location,
781 },
782
783 #[snafu(display("Invalid prefix: {}, key: {}", prefix, key))]
784 MismatchPrefix {
785 prefix: String,
786 key: String,
787 #[snafu(implicit)]
788 location: Location,
789 },
790
791 #[snafu(display("Failed to move values: {err_msg}"))]
792 MoveValues {
793 err_msg: String,
794 #[snafu(implicit)]
795 location: Location,
796 },
797
798 #[snafu(display("Failed to restore tombstone, target key already exists: {key}"))]
799 TombstoneTargetAlreadyExists {
800 key: String,
801 #[snafu(implicit)]
802 location: Location,
803 },
804
805 #[snafu(display("Failed to parse {} from utf8", name))]
806 FromUtf8 {
807 name: String,
808 #[snafu(source)]
809 error: std::string::FromUtf8Error,
810 #[snafu(implicit)]
811 location: Location,
812 },
813
814 #[snafu(display("Value not exists"))]
815 ValueNotExist {
816 #[snafu(implicit)]
817 location: Location,
818 },
819
820 #[snafu(display("Failed to get cache"))]
821 GetCache { source: Arc<Error> },
822
823 #[snafu(display(
824 "Failed to get latest cache value after {} attempts due to concurrent invalidation",
825 attempts
826 ))]
827 GetLatestCacheRetryExceeded {
828 attempts: usize,
829 #[snafu(implicit)]
830 location: Location,
831 },
832
833 #[cfg(feature = "pg_kvbackend")]
834 #[snafu(display("Failed to execute via Postgres, sql: {}", sql))]
835 PostgresExecution {
836 sql: String,
837 #[snafu(source)]
838 error: tokio_postgres::Error,
839 #[snafu(implicit)]
840 location: Location,
841 },
842
843 #[cfg(feature = "pg_kvbackend")]
844 #[snafu(display("Failed to create connection pool for Postgres"))]
845 CreatePostgresPool {
846 #[snafu(source)]
847 error: deadpool_postgres::CreatePoolError,
848 #[snafu(implicit)]
849 location: Location,
850 },
851
852 #[cfg(feature = "pg_kvbackend")]
853 #[snafu(display("Failed to get Postgres connection from pool: {}", reason))]
854 GetPostgresConnection {
855 reason: String,
856 #[snafu(implicit)]
857 location: Location,
858 },
859
860 #[cfg(feature = "pg_kvbackend")]
861 #[snafu(display("Failed to get Postgres client"))]
862 GetPostgresClient {
863 #[snafu(source)]
864 error: deadpool::managed::PoolError<tokio_postgres::Error>,
865 #[snafu(implicit)]
866 location: Location,
867 },
868
869 #[cfg(feature = "pg_kvbackend")]
870 #[snafu(display("Failed to {} Postgres transaction", operation))]
871 PostgresTransaction {
872 #[snafu(source)]
873 error: tokio_postgres::Error,
874 #[snafu(implicit)]
875 location: Location,
876 operation: String,
877 },
878
879 #[cfg(feature = "pg_kvbackend")]
880 #[snafu(display("Failed to setup PostgreSQL TLS configuration: {}", reason))]
881 PostgresTlsConfig {
882 reason: String,
883 #[snafu(implicit)]
884 location: Location,
885 },
886
887 #[snafu(display("Failed to load TLS certificate from path: {}", path))]
888 LoadTlsCertificate {
889 path: String,
890 #[snafu(source)]
891 error: std::io::Error,
892 #[snafu(implicit)]
893 location: Location,
894 },
895
896 #[cfg(feature = "pg_kvbackend")]
897 #[snafu(display("Invalid TLS configuration: {}", reason))]
898 InvalidTlsConfig {
899 reason: String,
900 #[snafu(implicit)]
901 location: Location,
902 },
903
904 #[cfg(feature = "mysql_kvbackend")]
905 #[snafu(display("Failed to execute via MySql, sql: {}", sql))]
906 MySqlExecution {
907 sql: String,
908 #[snafu(source)]
909 error: sqlx::Error,
910 #[snafu(implicit)]
911 location: Location,
912 },
913
914 #[cfg(feature = "mysql_kvbackend")]
915 #[snafu(display("Failed to create connection pool for MySql"))]
916 CreateMySqlPool {
917 #[snafu(source)]
918 error: sqlx::Error,
919 #[snafu(implicit)]
920 location: Location,
921 },
922
923 #[cfg(feature = "mysql_kvbackend")]
924 #[snafu(display("Failed to decode sql value"))]
925 DecodeSqlValue {
926 #[snafu(source)]
927 error: sqlx::error::Error,
928 #[snafu(implicit)]
929 location: Location,
930 },
931
932 #[cfg(feature = "mysql_kvbackend")]
933 #[snafu(display("Failed to acquire mysql client from pool"))]
934 AcquireMySqlClient {
935 #[snafu(source)]
936 error: sqlx::Error,
937 #[snafu(implicit)]
938 location: Location,
939 },
940
941 #[cfg(feature = "mysql_kvbackend")]
942 #[snafu(display("Failed to {} MySql transaction", operation))]
943 MySqlTransaction {
944 #[snafu(source)]
945 error: sqlx::Error,
946 #[snafu(implicit)]
947 location: Location,
948 operation: String,
949 },
950
951 #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
952 #[snafu(display("Rds transaction retry failed"))]
953 RdsTransactionRetryFailed {
954 #[snafu(implicit)]
955 location: Location,
956 },
957
958 #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
959 #[snafu(display("Sql execution timeout, sql: {}, duration: {:?}", sql, duration))]
960 SqlExecutionTimeout {
961 sql: String,
962 duration: std::time::Duration,
963 #[snafu(implicit)]
964 location: Location,
965 },
966
967 #[snafu(display(
968 "Datanode table info not found, table id: {}, datanode id: {}",
969 table_id,
970 datanode_id
971 ))]
972 DatanodeTableInfoNotFound {
973 datanode_id: DatanodeId,
974 table_id: TableId,
975 #[snafu(implicit)]
976 location: Location,
977 },
978
979 #[snafu(display("Invalid topic name prefix: {}", prefix))]
980 InvalidTopicNamePrefix {
981 prefix: String,
982 #[snafu(implicit)]
983 location: Location,
984 },
985
986 #[snafu(display("No leader found for table_id: {}", table_id))]
987 NoLeader {
988 table_id: TableId,
989 #[snafu(implicit)]
990 location: Location,
991 },
992
993 #[snafu(display(
994 "Procedure poison key already exists with a different value, key: {}, value: {}",
995 key,
996 value
997 ))]
998 ProcedurePoisonConflict {
999 key: String,
1000 value: String,
1001 #[snafu(implicit)]
1002 location: Location,
1003 },
1004
1005 #[snafu(display("Failed to put poison, table metadata may be corrupted"))]
1006 PutPoison {
1007 #[snafu(implicit)]
1008 location: Location,
1009 #[snafu(source)]
1010 source: common_procedure::error::Error,
1011 },
1012
1013 #[snafu(display("Invalid file path: {}", file_path))]
1014 InvalidFilePath {
1015 #[snafu(implicit)]
1016 location: Location,
1017 file_path: String,
1018 },
1019
1020 #[snafu(display("Failed to serialize flexbuffers"))]
1021 SerializeFlexbuffers {
1022 #[snafu(implicit)]
1023 location: Location,
1024 #[snafu(source)]
1025 error: flexbuffers::SerializationError,
1026 },
1027
1028 #[snafu(display("Failed to deserialize flexbuffers"))]
1029 DeserializeFlexbuffers {
1030 #[snafu(implicit)]
1031 location: Location,
1032 #[snafu(source)]
1033 error: flexbuffers::DeserializationError,
1034 },
1035
1036 #[snafu(display("Failed to read flexbuffers"))]
1037 ReadFlexbuffers {
1038 #[snafu(implicit)]
1039 location: Location,
1040 #[snafu(source)]
1041 error: flexbuffers::ReaderError,
1042 },
1043
1044 #[snafu(display("Invalid file name: {}", reason))]
1045 InvalidFileName {
1046 #[snafu(implicit)]
1047 location: Location,
1048 reason: String,
1049 },
1050
1051 #[snafu(display("Invalid file extension: {}", reason))]
1052 InvalidFileExtension {
1053 #[snafu(implicit)]
1054 location: Location,
1055 reason: String,
1056 },
1057
1058 #[snafu(display("Failed to write object, file path: {}", file_path))]
1059 WriteObject {
1060 #[snafu(implicit)]
1061 location: Location,
1062 file_path: String,
1063 #[snafu(source)]
1064 error: object_store::Error,
1065 },
1066
1067 #[snafu(display("Failed to read object, file path: {}", file_path))]
1068 ReadObject {
1069 #[snafu(implicit)]
1070 location: Location,
1071 file_path: String,
1072 #[snafu(source)]
1073 error: object_store::Error,
1074 },
1075
1076 #[snafu(display("Missing column ids"))]
1077 MissingColumnIds {
1078 #[snafu(implicit)]
1079 location: Location,
1080 },
1081
1082 #[snafu(display(
1083 "Missing column in column metadata: {}, table: {}, table_id: {}",
1084 column_name,
1085 table_name,
1086 table_id,
1087 ))]
1088 MissingColumnInColumnMetadata {
1089 column_name: String,
1090 #[snafu(implicit)]
1091 location: Location,
1092 table_name: String,
1093 table_id: TableId,
1094 },
1095
1096 #[snafu(display(
1097 "Mismatch column id: column_name: {}, column_id: {}, table: {}, table_id: {}",
1098 column_name,
1099 column_id,
1100 table_name,
1101 table_id,
1102 ))]
1103 MismatchColumnId {
1104 column_name: String,
1105 column_id: u32,
1106 #[snafu(implicit)]
1107 location: Location,
1108 table_name: String,
1109 table_id: TableId,
1110 },
1111
1112 #[snafu(display("Failed to convert column def, column: {}", column))]
1113 ConvertColumnDef {
1114 column: String,
1115 #[snafu(implicit)]
1116 location: Location,
1117 source: api::error::Error,
1118 },
1119
1120 #[snafu(display("Failed to convert time ranges"))]
1121 ConvertTimeRanges {
1122 #[snafu(implicit)]
1123 location: Location,
1124 source: api::error::Error,
1125 },
1126
1127 #[snafu(display(
1128 "Column metadata inconsistencies found in table: {}, table_id: {}",
1129 table_name,
1130 table_id
1131 ))]
1132 ColumnMetadataConflicts {
1133 table_name: String,
1134 table_id: TableId,
1135 },
1136
1137 #[snafu(display(
1138 "Column not found in column metadata, column_name: {}, column_id: {}",
1139 column_name,
1140 column_id
1141 ))]
1142 ColumnNotFound { column_name: String, column_id: u32 },
1143
1144 #[snafu(display(
1145 "Column id mismatch, column_name: {}, expected column_id: {}, actual column_id: {}",
1146 column_name,
1147 expected_column_id,
1148 actual_column_id
1149 ))]
1150 ColumnIdMismatch {
1151 column_name: String,
1152 expected_column_id: u32,
1153 actual_column_id: u32,
1154 },
1155
1156 #[snafu(display(
1157 "Timestamp column mismatch, expected column_name: {}, expected column_id: {}, actual column_name: {}, actual column_id: {}",
1158 expected_column_name,
1159 expected_column_id,
1160 actual_column_name,
1161 actual_column_id,
1162 ))]
1163 TimestampMismatch {
1164 expected_column_name: String,
1165 expected_column_id: u32,
1166 actual_column_name: String,
1167 actual_column_id: u32,
1168 },
1169
1170 #[snafu(display(
1171 "Partition expression of region {} is incompatible with the altered column '{}': the persisted bound can no longer be compared with the new column type {}",
1172 region_id,
1173 column,
1174 target_type
1175 ))]
1176 PartitionExprIncompatible {
1177 region_id: RegionId,
1178 column: String,
1179 target_type: String,
1180 #[snafu(implicit)]
1181 location: Location,
1182 },
1183
1184 #[cfg(feature = "enterprise")]
1185 #[snafu(display("Too large duration"))]
1186 TooLargeDuration {
1187 #[snafu(source)]
1188 error: prost_types::DurationError,
1189 #[snafu(implicit)]
1190 location: Location,
1191 },
1192
1193 #[cfg(feature = "enterprise")]
1194 #[snafu(display("Negative duration"))]
1195 NegativeDuration {
1196 #[snafu(source)]
1197 error: prost_types::DurationError,
1198 #[snafu(implicit)]
1199 location: Location,
1200 },
1201
1202 #[cfg(feature = "enterprise")]
1203 #[snafu(display("Missing interval field"))]
1204 MissingInterval {
1205 #[snafu(implicit)]
1206 location: Location,
1207 },
1208}
1209
1210pub type Result<T> = std::result::Result<T, Error>;
1211
1212impl ErrorExt for Error {
1213 fn status_code(&self) -> StatusCode {
1214 use Error::*;
1215 match self {
1216 Error::PartialRegion { response } => response
1217 .first_failure()
1218 .map_or(StatusCode::Internal, |(_, error)| error.status_code()),
1219 IllegalServerState { .. }
1220 | EtcdTxnOpResponse { .. }
1221 | EtcdFailed { .. }
1222 | EtcdTxnFailed { .. }
1223 | ConnectEtcd { .. }
1224 | MoveValues { .. }
1225 | TombstoneTargetAlreadyExists { .. }
1226 | GetCache { .. }
1227 | GetLatestCacheRetryExceeded { .. }
1228 | SerializeToJson { .. }
1229 | DeserializeFromJson { .. }
1230 | ElectionNoLeader { .. }
1231 | ElectionLeaderLeaseExpired { .. }
1232 | ElectionLeaderLeaseChanged { .. } => StatusCode::Internal,
1233
1234 NoLeader { .. } => StatusCode::TableUnavailable,
1235 ValueNotExist { .. }
1236 | ProcedurePoisonConflict { .. }
1237 | ProcedureStateReceiverNotFound { .. }
1238 | MissingColumnIds { .. }
1239 | MissingColumnInColumnMetadata { .. }
1240 | MismatchColumnId { .. }
1241 | ColumnMetadataConflicts { .. }
1242 | ColumnNotFound { .. }
1243 | ColumnIdMismatch { .. }
1244 | TimestampMismatch { .. } => StatusCode::Unexpected,
1245
1246 Unsupported { .. } | ReadOnlyKvBackend { .. } => StatusCode::Unsupported,
1247 WriteObject { .. } | ReadObject { .. } => StatusCode::StorageUnavailable,
1248
1249 SerdeJson { .. }
1250 | ParseOption { .. }
1251 | InvalidProtoMsg { .. }
1252 | InvalidMetadata { .. }
1253 | Unexpected { .. }
1254 | TableInfoNotFound { .. }
1255 | NextSequence { .. }
1256 | UnexpectedSequenceValue { .. }
1257 | InvalidHeartbeatResponse { .. }
1258 | EncodeJson { .. }
1259 | DecodeJson { .. }
1260 | PayloadNotExist { .. }
1261 | ConvertRawKey { .. }
1262 | DecodeProto { .. }
1263 | DecodePackedFileRefs { .. }
1264 | InvalidPackedFileRefs { .. }
1265 | BuildTableMeta { .. }
1266 | TableRouteNotFound { .. }
1267 | TableRepartNotFound { .. }
1268 | RegionOperatingRace { .. }
1269 | SerializeWalOptions { .. }
1270 | BuildKafkaClient { .. }
1271 | BuildKafkaCtrlClient { .. }
1272 | KafkaPartitionClient { .. }
1273 | ProduceRecord { .. }
1274 | CreateKafkaWalTopic { .. }
1275 | EmptyTopicPool { .. }
1276 | UnexpectedLogicalRouteTable { .. }
1277 | ProcedureOutput { .. }
1278 | FromUtf8 { .. }
1279 | MetadataCorruption { .. }
1280 | KafkaGetOffset { .. }
1281 | ReadFlexbuffers { .. }
1282 | SerializeFlexbuffers { .. }
1283 | DeserializeFlexbuffers { .. }
1284 | ConvertTimeRanges { .. } => StatusCode::Unexpected,
1285
1286 GetKvCache { .. } | CacheNotGet { .. } => StatusCode::Internal,
1287
1288 SchemaAlreadyExists { .. } => StatusCode::DatabaseAlreadyExists,
1289
1290 ProcedureNotFound { .. }
1291 | InvalidViewInfo { .. }
1292 | InvalidFlowOption { .. }
1293 | PrimaryKeyNotFound { .. }
1294 | EmptyKey { .. }
1295 | AlterLogicalTablesInvalidArguments { .. }
1296 | CreateLogicalTablesInvalidArguments { .. }
1297 | MismatchPrefix { .. }
1298 | TlsConfig { .. }
1299 | InvalidSetDatabaseOption { .. }
1300 | InvalidUnsetDatabaseOption { .. }
1301 | InvalidTopicNamePrefix { .. }
1302 | InvalidFileExtension { .. }
1303 | InvalidFileName { .. }
1304 | InvalidFlowRequestBody { .. }
1305 | InvalidFilePath { .. }
1306 | ConflictingSchemaOptions { .. }
1307 | PartitionExprIncompatible { .. } => StatusCode::InvalidArguments,
1308
1309 #[cfg(feature = "enterprise")]
1310 MissingInterval { .. } | NegativeDuration { .. } | TooLargeDuration { .. } => {
1311 StatusCode::InvalidArguments
1312 }
1313
1314 FlowNotFound { .. } => StatusCode::FlowNotFound,
1315 FlowRouteNotFound { .. } => StatusCode::Unexpected,
1316 FlowAlreadyExists { .. } => StatusCode::FlowAlreadyExists,
1317
1318 ViewNotFound { .. } | TableNotFound { .. } | RegionNotFound { .. } => {
1319 StatusCode::TableNotFound
1320 }
1321 ViewAlreadyExists { .. }
1322 | TableAlreadyExists { .. }
1323 | TableNameTombstoneConflict { .. } => StatusCode::TableAlreadyExists,
1324
1325 SubmitProcedure { source, .. }
1326 | QueryProcedure { source, .. }
1327 | WaitProcedure { source, .. }
1328 | StartProcedureManager { source, .. }
1329 | StopProcedureManager { source, .. } => source.status_code(),
1330 RegisterProcedureLoader { source, .. } => source.status_code(),
1331 External { source, .. } => source.status_code(),
1332 ResponseExceededSizeLimit { source, .. } => source.status_code(),
1333 OperateDatanode { source, .. } => source.status_code(),
1334 Table { source, .. } => source.status_code(),
1335 RetryLater { source, .. } => source.status_code(),
1336 AbortProcedure { source, .. } => source.status_code(),
1337 ConvertAlterTableRequest { source, .. } => source.status_code(),
1338 PutPoison { source, .. } => source.status_code(),
1339 ConvertColumnDef { source, .. } => source.status_code(),
1340 ProcedureStateReceiver { source, .. } => source.status_code(),
1341 RegisterRepartitionProcedureLoader { source, .. } => source.status_code(),
1342 CreateRepartitionProcedure { source, .. } => source.status_code(),
1343 PersistRepartitionGcRequirement { source, .. } => source.status_code(),
1344
1345 ParseProcedureId { .. }
1346 | InvalidNumTopics { .. }
1347 | SchemaNotFound { .. }
1348 | CatalogNotFound { .. }
1349 | InvalidNodeInfoKey { .. }
1350 | InvalidStatKey { .. }
1351 | ParseNum { .. }
1352 | InvalidRole { .. }
1353 | EmptyDdlTasks { .. } => StatusCode::InvalidArguments,
1354
1355 DdlTaskTimeout { .. } => StatusCode::Cancelled,
1358 LoadTlsCertificate { .. } => StatusCode::Internal,
1359
1360 #[cfg(feature = "pg_kvbackend")]
1361 PostgresExecution { .. }
1362 | CreatePostgresPool { .. }
1363 | GetPostgresConnection { .. }
1364 | GetPostgresClient { .. }
1365 | PostgresTransaction { .. }
1366 | PostgresTlsConfig { .. }
1367 | InvalidTlsConfig { .. } => StatusCode::Internal,
1368 #[cfg(feature = "mysql_kvbackend")]
1369 MySqlExecution { .. }
1370 | CreateMySqlPool { .. }
1371 | DecodeSqlValue { .. }
1372 | AcquireMySqlClient { .. }
1373 | MySqlTransaction { .. } => StatusCode::Internal,
1374 #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
1375 RdsTransactionRetryFailed { .. } | SqlExecutionTimeout { .. } => StatusCode::Internal,
1376 DatanodeTableInfoNotFound { .. } => StatusCode::Internal,
1377 }
1378 }
1379
1380 fn as_any(&self) -> &dyn std::any::Any {
1381 self
1382 }
1383
1384 fn retry_hint(&self) -> RetryHint {
1385 use Error::*;
1386
1387 match self {
1388 Error::PartialRegion { .. } => RetryHint::Retryable,
1389 RetryLater { source, .. } => match source.retry_hint() {
1390 RetryHint::RetryAfterRouteRefresh => RetryHint::RetryAfterRouteRefresh,
1391 RetryHint::Retryable | RetryHint::NonRetryable => RetryHint::Retryable,
1392 },
1393 GetLatestCacheRetryExceeded { .. }
1394 | NoLeader { .. }
1395 | ElectionNoLeader { .. }
1396 | ElectionLeaderLeaseExpired { .. }
1397 | ElectionLeaderLeaseChanged { .. } => RetryHint::Retryable,
1398 ConnectEtcd { error, .. } | EtcdFailed { error, .. } | EtcdTxnFailed { error, .. } => {
1399 retry_hint_from_etcd_error(error)
1400 }
1401 WriteObject { error, .. } | ReadObject { error, .. } => {
1402 retry_hint_from_opendal_error(error)
1403 }
1404 BuildKafkaClient { error, .. }
1405 | BuildKafkaCtrlClient { error, .. }
1406 | KafkaPartitionClient { error, .. }
1407 | KafkaGetOffset { error, .. }
1408 | ProduceRecord { error, .. }
1409 | CreateKafkaWalTopic { error, .. } => rskafka_client_error_to_retry_hint(error),
1410 SubmitProcedure { source, .. }
1411 | QueryProcedure { source, .. }
1412 | WaitProcedure { source, .. }
1413 | StartProcedureManager { source, .. }
1414 | StopProcedureManager { source, .. }
1415 | RegisterProcedureLoader { source, .. }
1416 | PutPoison { source, .. } => source.retry_hint(),
1417 External { source, .. }
1418 | ResponseExceededSizeLimit { source, .. }
1419 | OperateDatanode { source, .. }
1420 | RegisterRepartitionProcedureLoader { source, .. }
1421 | CreateRepartitionProcedure { source, .. }
1422 | PersistRepartitionGcRequirement { source, .. } => source.retry_hint(),
1423 AbortProcedure { .. } | ProcedureStateReceiver { .. } => RetryHint::NonRetryable,
1425 Table { source, .. } => source.retry_hint(),
1426 ConvertAlterTableRequest { source, .. } => source.retry_hint(),
1427 ConvertColumnDef { source, .. } => source.retry_hint(),
1428 GetCache { source, .. } => source.retry_hint(),
1429 #[cfg(feature = "pg_kvbackend")]
1430 PostgresExecution { error, .. } | PostgresTransaction { error, .. } => {
1431 retry_hint_from_postgres_error(error)
1432 }
1433 #[cfg(feature = "pg_kvbackend")]
1434 GetPostgresClient { error, .. } => retry_hint_from_postgres_pool_error(error),
1435 #[cfg(feature = "mysql_kvbackend")]
1436 MySqlExecution { error, .. }
1437 | CreateMySqlPool { error, .. }
1438 | AcquireMySqlClient { error, .. }
1439 | MySqlTransaction { error, .. } => retry_hint_from_sqlx_error(error),
1440 #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
1441 RdsTransactionRetryFailed { .. } | SqlExecutionTimeout { .. } => RetryHint::Retryable,
1442 _ => RetryHint::NonRetryable,
1443 }
1444 }
1445}
1446
1447impl Error {
1448 #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
1449 pub fn is_serialization_error(&self) -> bool {
1451 match self {
1452 #[cfg(feature = "pg_kvbackend")]
1453 Error::PostgresExecution { error, .. } | Error::PostgresTransaction { error, .. } => {
1454 retry_hint::is_postgres_serialization_error(error)
1455 }
1456 #[cfg(feature = "mysql_kvbackend")]
1457 Error::MySqlExecution { error, .. } | Error::MySqlTransaction { error, .. } => {
1458 retry_hint::is_mysql_serialization_error(error)
1459 }
1460 _ => false,
1461 }
1462 }
1463
1464 pub fn retry_later<E: ErrorExt + Send + Sync + 'static>(err: E) -> Error {
1466 Error::RetryLater {
1467 source: BoxedError::new(err),
1468 clean_poisons: false,
1469 }
1470 }
1471
1472 pub fn is_retry_later(&self) -> bool {
1474 matches!(
1475 self,
1476 Error::RetryLater { .. } | Error::GetLatestCacheRetryExceeded { .. }
1477 )
1478 }
1479
1480 pub fn need_clean_poisons(&self) -> bool {
1482 matches!(
1483 self,
1484 Error::AbortProcedure { clean_poisons, .. } if *clean_poisons
1485 ) || matches!(
1486 self,
1487 Error::RetryLater { clean_poisons, .. } if *clean_poisons
1488 )
1489 }
1490
1491 pub fn is_exceeded_size_limit(&self) -> bool {
1493 match self {
1494 Error::EtcdFailed {
1495 error: etcd_client::Error::GRpcStatus(status),
1496 ..
1497 } => status.code() == tonic::Code::OutOfRange,
1498 Error::ResponseExceededSizeLimit { .. } => true,
1499 _ => false,
1500 }
1501 }
1502}
1503
1504#[cfg(test)]
1505mod retry_hint_tests {
1506 use std::sync::Arc;
1507
1508 use common_error::mock::MockError;
1509 use snafu::ResultExt;
1510
1511 use super::*;
1512
1513 #[test]
1514 fn test_retry_later_hint_is_retryable() {
1515 let err = Error::retry_later(MockError::new(StatusCode::Internal));
1516
1517 assert_eq!(err.retry_hint(), RetryHint::Retryable);
1518 }
1519
1520 #[test]
1521 fn test_latest_cache_retry_exceeded_hint_is_retryable() {
1522 let err = GetLatestCacheRetryExceededSnafu { attempts: 3_usize }.build();
1523
1524 assert_eq!(err.retry_hint(), RetryHint::Retryable);
1525 }
1526
1527 #[test]
1528 fn test_leader_retry_hints() {
1529 for (error, expected) in [
1530 (
1531 NoLeaderSnafu { table_id: 1_u32 }.build(),
1532 RetryHint::Retryable,
1533 ),
1534 (ElectionNoLeaderSnafu.build(), RetryHint::Retryable),
1535 (
1536 ElectionLeaderLeaseExpiredSnafu.build(),
1537 RetryHint::Retryable,
1538 ),
1539 (
1540 ElectionLeaderLeaseChangedSnafu.build(),
1541 RetryHint::Retryable,
1542 ),
1543 ] {
1544 assert_eq!(error.retry_hint(), expected, "{error:?}");
1545 }
1546 }
1547
1548 #[test]
1549 fn test_procedure_error_retry_hints() {
1550 let source = common_procedure::Error::retry_later(MockError::new(StatusCode::Internal));
1551 let err = Err::<(), _>(source).context(PutPoisonSnafu).unwrap_err();
1552 assert_eq!(err.retry_hint(), RetryHint::Retryable);
1553
1554 let source = common_procedure::Error::retry_later(MockError::new(StatusCode::Internal));
1555 let err = Err::<(), _>(source)
1556 .context(ProcedureStateReceiverSnafu {
1557 procedure_id: ProcedureId::random(),
1558 })
1559 .unwrap_err();
1560 assert_eq!(err.retry_hint(), RetryHint::NonRetryable);
1561 }
1562
1563 #[test]
1564 fn test_get_cache_forwards_retry_hint() {
1565 let source = Arc::new(Error::retry_later(MockError::new(StatusCode::Internal)));
1566 let err = Error::GetCache { source };
1567
1568 assert_eq!(err.retry_hint(), RetryHint::Retryable);
1569 }
1570
1571 #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
1572 #[test]
1573 fn test_sql_execution_timeout_hint_is_retryable() {
1574 let err = SqlExecutionTimeoutSnafu {
1575 sql: "SELECT 1".to_string(),
1576 duration: std::time::Duration::from_secs(1),
1577 }
1578 .build();
1579
1580 assert_eq!(err.retry_hint(), RetryHint::Retryable);
1581 }
1582
1583 #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
1584 #[test]
1585 fn test_rds_transaction_retry_failed_hint_is_retryable() {
1586 let err = RdsTransactionRetryFailedSnafu.build();
1587
1588 assert_eq!(err.retry_hint(), RetryHint::Retryable);
1589 }
1590
1591 #[test]
1592 fn test_default_hint_is_non_retryable() {
1593 let err = UnexpectedSnafu {
1594 err_msg: "mock error",
1595 }
1596 .build();
1597
1598 assert_eq!(err.retry_hint(), RetryHint::NonRetryable);
1599 }
1600}