1use common_error::define_into_tonic_status;
16use common_error::ext::{BoxedError, ErrorExt, RetryHint};
17use common_error::status_code::StatusCode;
18use common_macro::stack_trace_debug;
19use common_meta::DatanodeId;
20use common_procedure::ProcedureId;
21use common_runtime::JoinError;
22use common_wal::kafka::rskafka_client_error_to_retry_hint;
23use snafu::{Location, Snafu};
24use store_api::storage::RegionId;
25use table::metadata::TableId;
26use tokio::sync::mpsc::error::SendError;
27use tonic::codegen::http;
28use uuid::Uuid;
29
30use crate::metasrv::SelectTarget;
31use crate::pubsub::Message;
32use crate::service::mailbox::Channel;
33
34#[derive(Snafu)]
35#[snafu(visibility(pub))]
36#[stack_trace_debug]
37pub enum Error {
38 #[snafu(display("Failed to choose items"))]
39 ChooseItems {
40 #[snafu(implicit)]
41 location: Location,
42 #[snafu(source)]
43 error: rand::distr::weighted::Error,
44 },
45
46 #[snafu(display("Exceeded deadline, operation: {}", operation))]
47 ExceededDeadline {
48 #[snafu(implicit)]
49 location: Location,
50 operation: String,
51 },
52
53 #[snafu(display("The target peer is unavailable temporally: {}", peer_id))]
54 PeerUnavailable {
55 #[snafu(implicit)]
56 location: Location,
57 peer_id: u64,
58 },
59
60 #[snafu(display("Failed to list active frontends"))]
61 ListActiveFrontends {
62 #[snafu(implicit)]
63 location: Location,
64 source: common_meta::error::Error,
65 },
66
67 #[snafu(display("Failed to list active datanodes"))]
68 ListActiveDatanodes {
69 #[snafu(implicit)]
70 location: Location,
71 source: common_meta::error::Error,
72 },
73
74 #[snafu(display("Failed to list active flownodes"))]
75 ListActiveFlownodes {
76 #[snafu(implicit)]
77 location: Location,
78 source: common_meta::error::Error,
79 },
80
81 #[snafu(display("No available frontend"))]
82 NoAvailableFrontend {
83 #[snafu(implicit)]
84 location: Location,
85 },
86
87 #[snafu(display("Another migration procedure is running for region: {}", region_id))]
88 MigrationRunning {
89 #[snafu(implicit)]
90 location: Location,
91 region_id: RegionId,
92 },
93
94 #[snafu(display(
95 "The region migration procedure is completed for region: {}, target_peer: {}",
96 region_id,
97 target_peer_id
98 ))]
99 RegionMigrated {
100 #[snafu(implicit)]
101 location: Location,
102 region_id: RegionId,
103 target_peer_id: u64,
104 },
105
106 #[snafu(display("The region migration procedure aborted, reason: {}", reason))]
107 MigrationAbort {
108 #[snafu(implicit)]
109 location: Location,
110 reason: String,
111 },
112
113 #[snafu(display(
114 "Another procedure is operating the region: {} on peer: {}",
115 region_id,
116 peer_id
117 ))]
118 RegionOperatingRace {
119 #[snafu(implicit)]
120 location: Location,
121 peer_id: DatanodeId,
122 region_id: RegionId,
123 },
124
125 #[snafu(display("Failed to init ddl manager"))]
126 InitDdlManager {
127 #[snafu(implicit)]
128 location: Location,
129 source: common_meta::error::Error,
130 },
131
132 #[snafu(display("Failed to init reconciliation manager"))]
133 InitReconciliationManager {
134 #[snafu(implicit)]
135 location: Location,
136 source: common_meta::error::Error,
137 },
138
139 #[snafu(display("Failed to create default catalog and schema"))]
140 InitMetadata {
141 #[snafu(implicit)]
142 location: Location,
143 source: common_meta::error::Error,
144 },
145
146 #[snafu(display("Failed to allocate next sequence number"))]
147 NextSequence {
148 #[snafu(implicit)]
149 location: Location,
150 source: common_meta::error::Error,
151 },
152
153 #[snafu(display("Failed to set next sequence number"))]
154 SetNextSequence {
155 #[snafu(implicit)]
156 location: Location,
157 source: common_meta::error::Error,
158 },
159
160 #[snafu(display("Failed to peek sequence number"))]
161 PeekSequence {
162 #[snafu(implicit)]
163 location: Location,
164 source: common_meta::error::Error,
165 },
166
167 #[snafu(display("Failed to start telemetry task"))]
168 StartTelemetryTask {
169 #[snafu(implicit)]
170 location: Location,
171 source: common_runtime::error::Error,
172 },
173
174 #[snafu(display("Failed to submit ddl task"))]
175 SubmitDdlTask {
176 #[snafu(implicit)]
177 location: Location,
178 source: common_meta::error::Error,
179 },
180
181 #[snafu(display("Failed to submit reconcile procedure"))]
182 SubmitReconcileProcedure {
183 #[snafu(implicit)]
184 location: Location,
185 source: common_meta::error::Error,
186 },
187
188 #[snafu(display("Failed to invalidate table cache"))]
189 InvalidateTableCache {
190 #[snafu(implicit)]
191 location: Location,
192 source: common_meta::error::Error,
193 },
194
195 #[snafu(display("Failed to list catalogs"))]
196 ListCatalogs {
197 #[snafu(implicit)]
198 location: Location,
199 source: BoxedError,
200 },
201
202 #[snafu(display("Failed to list {}'s schemas", catalog))]
203 ListSchemas {
204 #[snafu(implicit)]
205 location: Location,
206 catalog: String,
207 source: BoxedError,
208 },
209
210 #[snafu(display("Failed to list {}.{}'s tables", catalog, schema))]
211 ListTables {
212 #[snafu(implicit)]
213 location: Location,
214 catalog: String,
215 schema: String,
216 source: BoxedError,
217 },
218
219 #[snafu(display("Failed to join a future"))]
220 Join {
221 #[snafu(implicit)]
222 location: Location,
223 #[snafu(source)]
224 error: JoinError,
225 },
226
227 #[snafu(display(
228 "Failed to request {}, required: {}, but only {} available",
229 select_target,
230 required,
231 available
232 ))]
233 NoEnoughAvailableNode {
234 #[snafu(implicit)]
235 location: Location,
236 required: usize,
237 available: usize,
238 select_target: SelectTarget,
239 },
240
241 #[snafu(display("Failed to send shutdown signal"))]
242 SendShutdownSignal {
243 #[snafu(source)]
244 error: SendError<()>,
245 },
246
247 #[snafu(display("Failed to shutdown {} server", server))]
248 ShutdownServer {
249 #[snafu(implicit)]
250 location: Location,
251 source: servers::error::Error,
252 server: String,
253 },
254
255 #[snafu(display("Empty key is not allowed"))]
256 EmptyKey {
257 #[snafu(implicit)]
258 location: Location,
259 },
260
261 #[snafu(display("Failed to execute via Etcd"))]
262 EtcdFailed {
263 #[snafu(source)]
264 error: etcd_client::Error,
265 #[snafu(implicit)]
266 location: Location,
267 },
268
269 #[snafu(display("Failed to connect to Etcd"))]
270 ConnectEtcd {
271 #[snafu(source)]
272 error: etcd_client::Error,
273 #[snafu(implicit)]
274 location: Location,
275 },
276
277 #[snafu(display("Failed to read file: {}", path))]
278 FileIo {
279 #[snafu(source)]
280 error: std::io::Error,
281 #[snafu(implicit)]
282 location: Location,
283 path: String,
284 },
285
286 #[snafu(display("Failed to bind address {}", addr))]
287 TcpBind {
288 addr: String,
289 #[snafu(source)]
290 error: std::io::Error,
291 #[snafu(implicit)]
292 location: Location,
293 },
294
295 #[snafu(display("Failed to start gRPC server"))]
296 StartGrpc {
297 #[snafu(source)]
298 error: tonic::transport::Error,
299 #[snafu(implicit)]
300 location: Location,
301 },
302
303 #[snafu(display("Failed to start http server"))]
304 StartHttp {
305 #[snafu(implicit)]
306 location: Location,
307 source: servers::error::Error,
308 },
309
310 #[snafu(display("Failed to parse address {}", addr))]
311 ParseAddr {
312 addr: String,
313 #[snafu(source)]
314 error: std::net::AddrParseError,
315 },
316
317 #[snafu(display("Invalid lease key: {}", key))]
318 InvalidLeaseKey {
319 key: String,
320 #[snafu(implicit)]
321 location: Location,
322 },
323
324 #[snafu(display("Invalid datanode stat key: {}", key))]
325 InvalidStatKey {
326 key: String,
327 #[snafu(implicit)]
328 location: Location,
329 },
330
331 #[snafu(display("Invalid inactive region key: {}", key))]
332 InvalidInactiveRegionKey {
333 key: String,
334 #[snafu(implicit)]
335 location: Location,
336 },
337
338 #[snafu(display("Failed to parse lease key from utf8"))]
339 LeaseKeyFromUtf8 {
340 #[snafu(source)]
341 error: std::string::FromUtf8Error,
342 #[snafu(implicit)]
343 location: Location,
344 },
345
346 #[snafu(display("Failed to parse lease value from utf8"))]
347 LeaseValueFromUtf8 {
348 #[snafu(source)]
349 error: std::string::FromUtf8Error,
350 #[snafu(implicit)]
351 location: Location,
352 },
353
354 #[snafu(display("Failed to parse invalid region key from utf8"))]
355 InvalidRegionKeyFromUtf8 {
356 #[snafu(source)]
357 error: std::string::FromUtf8Error,
358 #[snafu(implicit)]
359 location: Location,
360 },
361
362 #[snafu(display("Failed to serialize to json: {}", input))]
363 SerializeToJson {
364 input: String,
365 #[snafu(source)]
366 error: serde_json::error::Error,
367 #[snafu(implicit)]
368 location: Location,
369 },
370
371 #[snafu(display("Failed to deserialize from json: {}", input))]
372 DeserializeFromJson {
373 input: String,
374 #[snafu(source)]
375 error: serde_json::error::Error,
376 #[snafu(implicit)]
377 location: Location,
378 },
379
380 #[snafu(display("Failed to serialize config"))]
381 SerializeConfig {
382 #[snafu(source)]
383 error: serde_json::error::Error,
384 #[snafu(implicit)]
385 location: Location,
386 },
387
388 #[snafu(display("Failed to parse number: {}", err_msg))]
389 ParseNum {
390 err_msg: String,
391 #[snafu(source)]
392 error: std::num::ParseIntError,
393 #[snafu(implicit)]
394 location: Location,
395 },
396
397 #[snafu(display("Failed to parse bool: {}", err_msg))]
398 ParseBool {
399 err_msg: String,
400 #[snafu(source)]
401 error: std::str::ParseBoolError,
402 #[snafu(implicit)]
403 location: Location,
404 },
405
406 #[snafu(display("Failed to downgrade region leader, region: {}", region_id))]
407 DowngradeLeader {
408 region_id: RegionId,
409 #[snafu(implicit)]
410 location: Location,
411 #[snafu(source)]
412 source: BoxedError,
413 },
414
415 #[snafu(display("Region's leader peer changed: {}", msg))]
416 LeaderPeerChanged {
417 msg: String,
418 #[snafu(implicit)]
419 location: Location,
420 },
421
422 #[snafu(display("Invalid arguments: {}", err_msg))]
423 InvalidArguments {
424 err_msg: String,
425 #[snafu(implicit)]
426 location: Location,
427 },
428
429 #[snafu(display("Manual GC is rejected because maintenance mode is enabled"))]
430 ManualGcRejectedByMaintenanceMode {
431 #[snafu(implicit)]
432 location: Location,
433 },
434
435 #[cfg(feature = "mysql_kvbackend")]
436 #[snafu(display("Failed to parse mysql url: {}", mysql_url))]
437 ParseMySqlUrl {
438 #[snafu(source)]
439 error: sqlx::error::Error,
440 mysql_url: String,
441 #[snafu(implicit)]
442 location: Location,
443 },
444
445 #[cfg(feature = "mysql_kvbackend")]
446 #[snafu(display("Failed to decode sql value"))]
447 DecodeSqlValue {
448 #[snafu(source)]
449 error: sqlx::error::Error,
450 #[snafu(implicit)]
451 location: Location,
452 },
453
454 #[snafu(display("Failed to find table route for {table_id}"))]
455 TableRouteNotFound {
456 table_id: TableId,
457 #[snafu(implicit)]
458 location: Location,
459 },
460
461 #[snafu(display("Failed to find table route for {region_id}"))]
462 RegionRouteNotFound {
463 region_id: RegionId,
464 #[snafu(implicit)]
465 location: Location,
466 },
467
468 #[snafu(display("Table info not found: {}", table_id))]
469 TableInfoNotFound {
470 table_id: TableId,
471 #[snafu(implicit)]
472 location: Location,
473 },
474
475 #[snafu(display("Datanode table not found: {}, datanode: {}", table_id, datanode_id))]
476 DatanodeTableNotFound {
477 table_id: TableId,
478 datanode_id: DatanodeId,
479 #[snafu(implicit)]
480 location: Location,
481 },
482
483 #[snafu(display("Metasrv has no leader at this moment"))]
484 NoLeader {
485 #[snafu(implicit)]
486 location: Location,
487 },
488
489 #[snafu(display("Leader lease expired"))]
490 LeaderLeaseExpired {
491 #[snafu(implicit)]
492 location: Location,
493 },
494
495 #[snafu(display("Leader lease changed during election"))]
496 LeaderLeaseChanged {
497 #[snafu(implicit)]
498 location: Location,
499 },
500
501 #[snafu(display("Table {} not found", name))]
502 TableNotFound {
503 name: String,
504 #[snafu(implicit)]
505 location: Location,
506 },
507
508 #[snafu(display("Unsupported selector type, {}", selector_type))]
509 UnsupportedSelectorType {
510 selector_type: String,
511 #[snafu(implicit)]
512 location: Location,
513 },
514
515 #[snafu(display("Unexpected, violated: {violated}"))]
516 Unexpected {
517 violated: String,
518 #[snafu(implicit)]
519 location: Location,
520 },
521
522 #[snafu(display("Failed to create gRPC channel"))]
523 CreateChannel {
524 #[snafu(implicit)]
525 location: Location,
526 source: common_grpc::error::Error,
527 },
528
529 #[snafu(display("Failed to batch get KVs from leader's in_memory kv store"))]
530 BatchGet {
531 #[snafu(source)]
532 error: tonic::Status,
533 #[snafu(implicit)]
534 location: Location,
535 },
536
537 #[snafu(display("Failed to batch range KVs from leader's in_memory kv store"))]
538 Range {
539 #[snafu(source)]
540 error: tonic::Status,
541 #[snafu(implicit)]
542 location: Location,
543 },
544
545 #[snafu(display("Response header not found"))]
546 ResponseHeaderNotFound {
547 #[snafu(implicit)]
548 location: Location,
549 },
550
551 #[snafu(display("The requested meta node is not leader, node addr: {}", node_addr))]
552 IsNotLeader {
553 node_addr: String,
554 #[snafu(implicit)]
555 location: Location,
556 },
557
558 #[snafu(display("Invalid http body"))]
559 InvalidHttpBody {
560 #[snafu(source)]
561 error: http::Error,
562 #[snafu(implicit)]
563 location: Location,
564 },
565
566 #[snafu(display(
567 "The number of retries for the grpc call {} exceeded the limit, {}",
568 func_name,
569 retry_num
570 ))]
571 ExceededRetryLimit {
572 func_name: String,
573 retry_num: usize,
574 #[snafu(implicit)]
575 location: Location,
576 },
577
578 #[snafu(display("Invalid utf-8 value"))]
579 InvalidUtf8Value {
580 #[snafu(source)]
581 error: std::string::FromUtf8Error,
582 #[snafu(implicit)]
583 location: Location,
584 },
585
586 #[snafu(display("Missing required parameter, param: {:?}", param))]
587 MissingRequiredParameter { param: String },
588
589 #[snafu(display("Failed to start procedure manager"))]
590 StartProcedureManager {
591 #[snafu(implicit)]
592 location: Location,
593 source: common_procedure::Error,
594 },
595
596 #[snafu(display("Failed to stop procedure manager"))]
597 StopProcedureManager {
598 #[snafu(implicit)]
599 location: Location,
600 source: common_procedure::Error,
601 },
602
603 #[snafu(display("Failed to wait procedure done"))]
604 WaitProcedure {
605 #[snafu(implicit)]
606 location: Location,
607 source: common_procedure::Error,
608 },
609
610 #[snafu(display("Failed to query procedure state"))]
611 QueryProcedure {
612 #[snafu(implicit)]
613 location: Location,
614 source: common_procedure::Error,
615 },
616
617 #[snafu(display("Procedure not found: {pid}"))]
618 ProcedureNotFound {
619 #[snafu(implicit)]
620 location: Location,
621 pid: String,
622 },
623
624 #[snafu(display("Failed to submit procedure"))]
625 SubmitProcedure {
626 #[snafu(implicit)]
627 location: Location,
628 source: common_procedure::Error,
629 },
630
631 #[snafu(display("A prune task for topic {} is already running", topic))]
632 PruneTaskAlreadyRunning {
633 topic: String,
634 #[snafu(implicit)]
635 location: Location,
636 },
637
638 #[snafu(display("Schema already exists, name: {schema_name}"))]
639 SchemaAlreadyExists {
640 schema_name: String,
641 #[snafu(implicit)]
642 location: Location,
643 },
644
645 #[snafu(display("Table already exists: {table_name}"))]
646 TableAlreadyExists {
647 table_name: String,
648 #[snafu(implicit)]
649 location: Location,
650 },
651
652 #[snafu(display("Pusher not found: {pusher_id}"))]
653 PusherNotFound {
654 pusher_id: String,
655 #[snafu(implicit)]
656 location: Location,
657 },
658
659 #[snafu(display("Failed to push message: {err_msg}"))]
660 PushMessage {
661 err_msg: String,
662 #[snafu(implicit)]
663 location: Location,
664 },
665
666 #[snafu(display("Mailbox already closed: {id}"))]
667 MailboxClosed {
668 id: u64,
669 #[snafu(implicit)]
670 location: Location,
671 },
672
673 #[snafu(display("Mailbox timeout: {id}"))]
674 MailboxTimeout {
675 id: u64,
676 #[snafu(implicit)]
677 location: Location,
678 },
679
680 #[snafu(display("Mailbox receiver got an error: {id}, {err_msg}"))]
681 MailboxReceiver {
682 id: u64,
683 err_msg: String,
684 #[snafu(implicit)]
685 location: Location,
686 },
687
688 #[snafu(display("Mailbox channel closed: {channel}"))]
689 MailboxChannelClosed {
690 channel: Channel,
691 #[snafu(implicit)]
692 location: Location,
693 },
694
695 #[snafu(display("Missing request header"))]
696 MissingRequestHeader {
697 #[snafu(implicit)]
698 location: Location,
699 },
700
701 #[snafu(display("Failed to register procedure loader, type name: {}", type_name))]
702 RegisterProcedureLoader {
703 type_name: String,
704 #[snafu(implicit)]
705 location: Location,
706 source: common_procedure::error::Error,
707 },
708
709 #[snafu(display(
710 "Received unexpected instruction reply, mailbox message: {}, reason: {}",
711 mailbox_message,
712 reason
713 ))]
714 UnexpectedInstructionReply {
715 mailbox_message: String,
716 reason: String,
717 #[snafu(implicit)]
718 location: Location,
719 },
720
721 #[snafu(display("Expected to retry later, reason: {}", reason))]
722 RetryLater {
723 reason: String,
724 #[snafu(implicit)]
725 location: Location,
726 },
727
728 #[snafu(display("Expected to retry later, reason: {}", reason))]
729 RetryLaterWithSource {
730 reason: String,
731 #[snafu(implicit)]
732 location: Location,
733 source: BoxedError,
734 },
735
736 #[snafu(display("Failed to convert proto data"))]
737 ConvertProtoData {
738 #[snafu(implicit)]
739 location: Location,
740 source: common_meta::error::Error,
741 },
742
743 #[snafu(display("Other error"))]
746 Other {
747 source: BoxedError,
748 #[snafu(implicit)]
749 location: Location,
750 },
751
752 #[snafu(display("Table metadata manager error"))]
753 TableMetadataManager {
754 source: common_meta::error::Error,
755 #[snafu(implicit)]
756 location: Location,
757 },
758
759 #[snafu(display("Runtime switch manager error"))]
760 RuntimeSwitchManager {
761 source: common_meta::error::Error,
762 #[snafu(implicit)]
763 location: Location,
764 },
765
766 #[snafu(display("Keyvalue backend error"))]
767 KvBackend {
768 source: common_meta::error::Error,
769 #[snafu(implicit)]
770 location: Location,
771 },
772
773 #[snafu(display("Failed to publish message"))]
774 PublishMessage {
775 #[snafu(source)]
776 error: SendError<Message>,
777 #[snafu(implicit)]
778 location: Location,
779 },
780
781 #[snafu(display("Too many partitions"))]
782 TooManyPartitions {
783 #[snafu(implicit)]
784 location: Location,
785 },
786
787 #[snafu(display("Failed to create repartition subtasks"))]
788 RepartitionCreateSubtasks {
789 source: partition::error::Error,
790 #[snafu(implicit)]
791 location: Location,
792 },
793
794 #[snafu(display(
795 "Source partition expression '{}' does not match any existing region",
796 expr
797 ))]
798 RepartitionSourceExprMismatch {
799 expr: String,
800 #[snafu(implicit)]
801 location: Location,
802 },
803
804 #[snafu(display(
805 "Failed to get the state receiver for repartition subprocedure {}",
806 procedure_id
807 ))]
808 RepartitionSubprocedureStateReceiver {
809 procedure_id: ProcedureId,
810 #[snafu(source)]
811 source: common_procedure::Error,
812 #[snafu(implicit)]
813 location: Location,
814 },
815
816 #[snafu(display("Unsupported operation {}", operation))]
817 Unsupported {
818 operation: String,
819 #[snafu(implicit)]
820 location: Location,
821 },
822
823 #[snafu(display("Unexpected table route type: {}", err_msg))]
824 UnexpectedLogicalRouteTable {
825 #[snafu(implicit)]
826 location: Location,
827 err_msg: String,
828 source: common_meta::error::Error,
829 },
830
831 #[snafu(display("Failed to save cluster info"))]
832 SaveClusterInfo {
833 #[snafu(implicit)]
834 location: Location,
835 source: common_meta::error::Error,
836 },
837
838 #[snafu(display("Invalid cluster info format"))]
839 InvalidClusterInfoFormat {
840 #[snafu(implicit)]
841 location: Location,
842 source: common_meta::error::Error,
843 },
844
845 #[snafu(display("Invalid datanode stat format"))]
846 InvalidDatanodeStatFormat {
847 #[snafu(implicit)]
848 location: Location,
849 source: common_meta::error::Error,
850 },
851
852 #[snafu(display("Invalid node info format"))]
853 InvalidNodeInfoFormat {
854 #[snafu(implicit)]
855 location: Location,
856 source: common_meta::error::Error,
857 },
858
859 #[snafu(display("Failed to serialize options to TOML"))]
860 TomlFormat {
861 #[snafu(implicit)]
862 location: Location,
863 #[snafu(source(from(common_config::error::Error, Box::new)))]
864 source: Box<common_config::error::Error>,
865 },
866
867 #[cfg(feature = "pg_kvbackend")]
868 #[snafu(display("Failed to execute via postgres, sql: {}", sql))]
869 PostgresExecution {
870 #[snafu(source)]
871 error: tokio_postgres::Error,
872 sql: String,
873 #[snafu(implicit)]
874 location: Location,
875 },
876
877 #[cfg(feature = "pg_kvbackend")]
878 #[snafu(display("Failed to get Postgres client"))]
879 GetPostgresClient {
880 #[snafu(implicit)]
881 location: Location,
882 #[snafu(source)]
883 error: deadpool::managed::PoolError<tokio_postgres::Error>,
884 },
885
886 #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
887 #[snafu(display("Sql execution timeout, sql: {}, duration: {:?}", sql, duration))]
888 SqlExecutionTimeout {
889 #[snafu(implicit)]
890 location: Location,
891 sql: String,
892 duration: std::time::Duration,
893 },
894
895 #[cfg(feature = "pg_kvbackend")]
896 #[snafu(display("Failed to create connection pool for Postgres"))]
897 CreatePostgresPool {
898 #[snafu(source)]
899 error: deadpool_postgres::CreatePoolError,
900 #[snafu(implicit)]
901 location: Location,
902 },
903
904 #[cfg(feature = "pg_kvbackend")]
905 #[snafu(display("Failed to get connection from Postgres pool: {}", reason))]
906 GetPostgresConnection {
907 reason: String,
908 #[snafu(implicit)]
909 location: Location,
910 },
911
912 #[cfg(feature = "mysql_kvbackend")]
913 #[snafu(display("Failed to execute via mysql, sql: {}", sql))]
914 MySqlExecution {
915 #[snafu(source)]
916 error: sqlx::Error,
917 #[snafu(implicit)]
918 location: Location,
919 sql: String,
920 },
921
922 #[cfg(feature = "mysql_kvbackend")]
923 #[snafu(display("Failed to create mysql pool"))]
924 CreateMySqlPool {
925 #[snafu(source)]
926 error: sqlx::Error,
927 #[snafu(implicit)]
928 location: Location,
929 },
930
931 #[cfg(feature = "mysql_kvbackend")]
932 #[snafu(display("Failed to acquire mysql client from pool"))]
933 AcquireMySqlClient {
934 #[snafu(source)]
935 error: sqlx::Error,
936 #[snafu(implicit)]
937 location: Location,
938 },
939
940 #[snafu(display("Handler not found: {}", name))]
941 HandlerNotFound {
942 name: String,
943 #[snafu(implicit)]
944 location: Location,
945 },
946
947 #[snafu(display("Flow state handler error"))]
948 FlowStateHandler {
949 #[snafu(implicit)]
950 location: Location,
951 source: common_meta::error::Error,
952 },
953
954 #[snafu(display("Failed to build wal provider"))]
955 BuildWalProvider {
956 #[snafu(implicit)]
957 location: Location,
958 source: common_meta::error::Error,
959 },
960
961 #[snafu(display("Failed to build kafka client."))]
962 BuildKafkaClient {
963 #[snafu(implicit)]
964 location: Location,
965 #[snafu(source)]
966 error: common_meta::error::Error,
967 },
968
969 #[snafu(display(
970 "Failed to build a Kafka partition client, topic: {}, partition: {}",
971 topic,
972 partition
973 ))]
974 BuildPartitionClient {
975 topic: String,
976 partition: i32,
977 #[snafu(implicit)]
978 location: Location,
979 #[snafu(source)]
980 error: rskafka::client::error::Error,
981 },
982
983 #[snafu(display(
984 "Failed to delete records from Kafka, topic: {}, partition: {}, offset: {}",
985 topic,
986 partition,
987 offset
988 ))]
989 DeleteRecords {
990 #[snafu(implicit)]
991 location: Location,
992 #[snafu(source)]
993 error: rskafka::client::error::Error,
994 topic: String,
995 partition: i32,
996 offset: u64,
997 },
998
999 #[snafu(display("Failed to get offset from Kafka, topic: {}", topic))]
1000 GetOffset {
1001 topic: String,
1002 #[snafu(implicit)]
1003 location: Location,
1004 #[snafu(source)]
1005 error: rskafka::client::error::Error,
1006 },
1007
1008 #[snafu(display("Failed to update the TopicNameValue in kvbackend, topic: {}", topic))]
1009 UpdateTopicNameValue {
1010 topic: String,
1011 #[snafu(implicit)]
1012 location: Location,
1013 #[snafu(source)]
1014 source: common_meta::error::Error,
1015 },
1016
1017 #[snafu(display("Failed to build tls options"))]
1018 BuildTlsOptions {
1019 #[snafu(implicit)]
1020 location: Location,
1021 #[snafu(source)]
1022 source: common_meta::error::Error,
1023 },
1024
1025 #[snafu(display(
1026 "Repartition group {} source region missing, region id: {}",
1027 group_id,
1028 region_id
1029 ))]
1030 RepartitionSourceRegionMissing {
1031 group_id: Uuid,
1032 region_id: RegionId,
1033 #[snafu(implicit)]
1034 location: Location,
1035 },
1036
1037 #[snafu(display(
1038 "Repartition group {} target region missing, region id: {}",
1039 group_id,
1040 region_id
1041 ))]
1042 RepartitionTargetRegionMissing {
1043 group_id: Uuid,
1044 region_id: RegionId,
1045 #[snafu(implicit)]
1046 location: Location,
1047 },
1048
1049 #[snafu(display("Failed to serialize partition expression"))]
1050 SerializePartitionExpr {
1051 #[snafu(source)]
1052 source: partition::error::Error,
1053 #[snafu(implicit)]
1054 location: Location,
1055 },
1056
1057 #[snafu(display("Failed to deserialize partition expression"))]
1058 DeserializePartitionExpr {
1059 #[snafu(source)]
1060 source: partition::error::Error,
1061 #[snafu(implicit)]
1062 location: Location,
1063 },
1064
1065 #[snafu(display("Empty partition expression"))]
1066 EmptyPartitionExpr {
1067 #[snafu(implicit)]
1068 location: Location,
1069 },
1070
1071 #[snafu(display(
1072 "Partition expression mismatch, region id: {}, expected: {}, actual: {}",
1073 region_id,
1074 expected,
1075 actual
1076 ))]
1077 PartitionExprMismatch {
1078 region_id: RegionId,
1079 expected: String,
1080 actual: String,
1081 #[snafu(implicit)]
1082 location: Location,
1083 },
1084
1085 #[snafu(display("Failed to allocate regions for table: {}", table_id))]
1086 AllocateRegions {
1087 #[snafu(implicit)]
1088 location: Location,
1089 table_id: TableId,
1090 #[snafu(source)]
1091 source: common_meta::error::Error,
1092 },
1093
1094 #[snafu(display("Failed to deallocate regions for table: {}", table_id))]
1095 DeallocateRegions {
1096 #[snafu(implicit)]
1097 location: Location,
1098 table_id: TableId,
1099 #[snafu(source)]
1100 source: common_meta::error::Error,
1101 },
1102
1103 #[snafu(display("Failed to build create request for table: {}", table_id))]
1104 BuildCreateRequest {
1105 #[snafu(implicit)]
1106 location: Location,
1107 table_id: TableId,
1108 #[snafu(source)]
1109 source: common_meta::error::Error,
1110 },
1111
1112 #[snafu(display("Failed to allocate region routes for table: {}", table_id))]
1113 AllocateRegionRoutes {
1114 #[snafu(implicit)]
1115 location: Location,
1116 table_id: TableId,
1117 #[snafu(source)]
1118 source: common_meta::error::Error,
1119 },
1120
1121 #[snafu(display("Failed to allocate wal options for table: {}", table_id))]
1122 AllocateWalOptions {
1123 #[snafu(implicit)]
1124 location: Location,
1125 table_id: TableId,
1126 #[snafu(source)]
1127 source: common_meta::error::Error,
1128 },
1129}
1130
1131impl Error {
1132 pub fn is_retryable(&self) -> bool {
1134 self.retry_hint().is_retryable()
1135 }
1136}
1137
1138pub type Result<T> = std::result::Result<T, Error>;
1139
1140define_into_tonic_status!(Error);
1141
1142impl ErrorExt for Error {
1143 fn status_code(&self) -> StatusCode {
1144 match self {
1145 Error::EtcdFailed { .. }
1146 | Error::ConnectEtcd { .. }
1147 | Error::FileIo { .. }
1148 | Error::TcpBind { .. }
1149 | Error::SerializeConfig { .. }
1150 | Error::SerializeToJson { .. }
1151 | Error::DeserializeFromJson { .. }
1152 | Error::NoLeader { .. }
1153 | Error::LeaderLeaseExpired { .. }
1154 | Error::LeaderLeaseChanged { .. }
1155 | Error::CreateChannel { .. }
1156 | Error::BatchGet { .. }
1157 | Error::Range { .. }
1158 | Error::ResponseHeaderNotFound { .. }
1159 | Error::InvalidHttpBody { .. }
1160 | Error::ExceededRetryLimit { .. }
1161 | Error::SendShutdownSignal { .. }
1162 | Error::PushMessage { .. }
1163 | Error::MailboxClosed { .. }
1164 | Error::MailboxReceiver { .. }
1165 | Error::StartGrpc { .. }
1166 | Error::PublishMessage { .. }
1167 | Error::Join { .. }
1168 | Error::ChooseItems { .. }
1169 | Error::FlowStateHandler { .. }
1170 | Error::BuildWalProvider { .. }
1171 | Error::BuildPartitionClient { .. }
1172 | Error::BuildKafkaClient { .. } => StatusCode::Internal,
1173
1174 Error::DeleteRecords { .. }
1175 | Error::GetOffset { .. }
1176 | Error::PeerUnavailable { .. }
1177 | Error::PusherNotFound { .. } => StatusCode::Unexpected,
1178 Error::MailboxTimeout { .. } | Error::ExceededDeadline { .. } => StatusCode::Cancelled,
1179 Error::PruneTaskAlreadyRunning { .. }
1180 | Error::RetryLater { .. }
1181 | Error::MailboxChannelClosed { .. }
1182 | Error::IsNotLeader { .. } => StatusCode::IllegalState,
1183 Error::RetryLaterWithSource { source, .. } => source.status_code(),
1184 Error::SerializePartitionExpr { source, .. }
1185 | Error::DeserializePartitionExpr { source, .. } => source.status_code(),
1186
1187 Error::Unsupported { .. } => StatusCode::Unsupported,
1188
1189 Error::SchemaAlreadyExists { .. } => StatusCode::DatabaseAlreadyExists,
1190
1191 Error::TableAlreadyExists { .. } => StatusCode::TableAlreadyExists,
1192 Error::EmptyKey { .. }
1193 | Error::MissingRequiredParameter { .. }
1194 | Error::MissingRequestHeader { .. }
1195 | Error::InvalidLeaseKey { .. }
1196 | Error::InvalidStatKey { .. }
1197 | Error::InvalidInactiveRegionKey { .. }
1198 | Error::ParseNum { .. }
1199 | Error::ParseBool { .. }
1200 | Error::ParseAddr { .. }
1201 | Error::UnsupportedSelectorType { .. }
1202 | Error::InvalidArguments { .. }
1203 | Error::ManualGcRejectedByMaintenanceMode { .. }
1204 | Error::ProcedureNotFound { .. }
1205 | Error::TooManyPartitions { .. }
1206 | Error::TomlFormat { .. }
1207 | Error::HandlerNotFound { .. }
1208 | Error::LeaderPeerChanged { .. }
1209 | Error::RepartitionSourceRegionMissing { .. }
1210 | Error::RepartitionTargetRegionMissing { .. }
1211 | Error::PartitionExprMismatch { .. }
1212 | Error::RepartitionSourceExprMismatch { .. }
1213 | Error::EmptyPartitionExpr { .. } => StatusCode::InvalidArguments,
1214 Error::LeaseKeyFromUtf8 { .. }
1215 | Error::LeaseValueFromUtf8 { .. }
1216 | Error::InvalidRegionKeyFromUtf8 { .. }
1217 | Error::TableRouteNotFound { .. }
1218 | Error::TableInfoNotFound { .. }
1219 | Error::DatanodeTableNotFound { .. }
1220 | Error::InvalidUtf8Value { .. }
1221 | Error::UnexpectedInstructionReply { .. }
1222 | Error::Unexpected { .. }
1223 | Error::RegionOperatingRace { .. }
1224 | Error::RegionRouteNotFound { .. }
1225 | Error::MigrationAbort { .. }
1226 | Error::MigrationRunning { .. }
1227 | Error::RegionMigrated { .. } => StatusCode::Unexpected,
1228 Error::TableNotFound { .. } => StatusCode::TableNotFound,
1229 Error::SaveClusterInfo { source, .. }
1230 | Error::InvalidClusterInfoFormat { source, .. }
1231 | Error::InvalidDatanodeStatFormat { source, .. }
1232 | Error::InvalidNodeInfoFormat { source, .. } => source.status_code(),
1233 Error::InvalidateTableCache { source, .. } => source.status_code(),
1234 Error::SubmitProcedure { source, .. }
1235 | Error::WaitProcedure { source, .. }
1236 | Error::QueryProcedure { source, .. } => source.status_code(),
1237 Error::ShutdownServer { source, .. } | Error::StartHttp { source, .. } => {
1238 source.status_code()
1239 }
1240 Error::StartProcedureManager { source, .. }
1241 | Error::StopProcedureManager { source, .. } => source.status_code(),
1242
1243 Error::ListCatalogs { source, .. }
1244 | Error::ListSchemas { source, .. }
1245 | Error::ListTables { source, .. } => source.status_code(),
1246 Error::StartTelemetryTask { source, .. } => source.status_code(),
1247
1248 Error::NextSequence { source, .. }
1249 | Error::SetNextSequence { source, .. }
1250 | Error::PeekSequence { source, .. } => source.status_code(),
1251 Error::DowngradeLeader { source, .. } => source.status_code(),
1252 Error::RegisterProcedureLoader { source, .. } => source.status_code(),
1253 Error::SubmitDdlTask { source, .. }
1254 | Error::SubmitReconcileProcedure { source, .. } => source.status_code(),
1255 Error::ConvertProtoData { source, .. }
1256 | Error::TableMetadataManager { source, .. }
1257 | Error::RuntimeSwitchManager { source, .. }
1258 | Error::KvBackend { source, .. }
1259 | Error::UnexpectedLogicalRouteTable { source, .. }
1260 | Error::UpdateTopicNameValue { source, .. } => source.status_code(),
1261 Error::ListActiveFrontends { source, .. }
1262 | Error::ListActiveDatanodes { source, .. }
1263 | Error::ListActiveFlownodes { source, .. } => source.status_code(),
1264 Error::NoAvailableFrontend { .. } => StatusCode::IllegalState,
1265
1266 Error::InitMetadata { source, .. }
1267 | Error::InitDdlManager { source, .. }
1268 | Error::InitReconciliationManager { source, .. } => source.status_code(),
1269
1270 Error::BuildTlsOptions { source, .. } => source.status_code(),
1271 Error::Other { source, .. } => source.status_code(),
1272 Error::RepartitionCreateSubtasks { source, .. } => source.status_code(),
1273 Error::RepartitionSubprocedureStateReceiver { source, .. } => source.status_code(),
1274 Error::AllocateRegions { source, .. } => source.status_code(),
1275 Error::DeallocateRegions { source, .. } => source.status_code(),
1276 Error::AllocateRegionRoutes { source, .. } => source.status_code(),
1277 Error::AllocateWalOptions { source, .. } => source.status_code(),
1278 Error::BuildCreateRequest { source, .. } => source.status_code(),
1279 Error::NoEnoughAvailableNode { .. } => StatusCode::RuntimeResourcesExhausted,
1280
1281 #[cfg(feature = "pg_kvbackend")]
1282 Error::CreatePostgresPool { .. }
1283 | Error::GetPostgresClient { .. }
1284 | Error::GetPostgresConnection { .. }
1285 | Error::PostgresExecution { .. } => StatusCode::Internal,
1286 #[cfg(feature = "mysql_kvbackend")]
1287 Error::MySqlExecution { .. }
1288 | Error::CreateMySqlPool { .. }
1289 | Error::ParseMySqlUrl { .. }
1290 | Error::DecodeSqlValue { .. }
1291 | Error::AcquireMySqlClient { .. } => StatusCode::Internal,
1292 #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
1293 Error::SqlExecutionTimeout { .. } => StatusCode::Internal,
1294 }
1295 }
1296
1297 fn as_any(&self) -> &dyn std::any::Any {
1298 self
1299 }
1300
1301 fn retry_hint(&self) -> RetryHint {
1302 match self {
1303 Error::RetryLater { .. }
1304 | Error::RetryLaterWithSource { .. }
1305 | Error::MailboxTimeout { .. }
1306 | Error::NoEnoughAvailableNode { .. }
1307 | Error::NoLeader { .. }
1308 | Error::LeaderLeaseExpired { .. }
1309 | Error::LeaderLeaseChanged { .. }
1310 | Error::PeerUnavailable { .. } => RetryHint::Retryable,
1311
1312 Error::ConnectEtcd { error, .. } | Error::EtcdFailed { error, .. } => {
1313 common_meta::error::retry_hint_from_etcd_error(error)
1314 }
1315
1316 #[cfg(feature = "pg_kvbackend")]
1317 Error::PostgresExecution { error, .. } => {
1318 common_meta::error::retry_hint_from_postgres_error(error)
1319 }
1320 #[cfg(feature = "pg_kvbackend")]
1321 Error::GetPostgresClient { error, .. } => {
1322 common_meta::error::retry_hint_from_postgres_pool_error(error)
1323 }
1324 #[cfg(feature = "mysql_kvbackend")]
1325 Error::MySqlExecution { error, .. }
1326 | Error::CreateMySqlPool { error, .. }
1327 | Error::AcquireMySqlClient { error, .. } => {
1328 common_meta::error::retry_hint_from_sqlx_error(error)
1329 }
1330 #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
1331 Error::SqlExecutionTimeout { .. } => RetryHint::Retryable,
1332
1333 Error::ListActiveFrontends { source, .. }
1334 | Error::ListActiveDatanodes { source, .. }
1335 | Error::ListActiveFlownodes { source, .. }
1336 | Error::InitDdlManager { source, .. }
1337 | Error::InitReconciliationManager { source, .. }
1338 | Error::InitMetadata { source, .. }
1339 | Error::NextSequence { source, .. }
1340 | Error::SetNextSequence { source, .. }
1341 | Error::PeekSequence { source, .. }
1342 | Error::SubmitDdlTask { source, .. }
1343 | Error::SubmitReconcileProcedure { source, .. }
1344 | Error::InvalidateTableCache { source, .. }
1345 | Error::ConvertProtoData { source, .. }
1346 | Error::TableMetadataManager { source, .. }
1347 | Error::RuntimeSwitchManager { source, .. }
1348 | Error::KvBackend { source, .. }
1349 | Error::UnexpectedLogicalRouteTable { source, .. }
1350 | Error::SaveClusterInfo { source, .. }
1351 | Error::InvalidClusterInfoFormat { source, .. }
1352 | Error::InvalidDatanodeStatFormat { source, .. }
1353 | Error::InvalidNodeInfoFormat { source, .. }
1354 | Error::FlowStateHandler { source, .. }
1355 | Error::BuildWalProvider { source, .. }
1356 | Error::BuildKafkaClient { error: source, .. }
1357 | Error::UpdateTopicNameValue { source, .. }
1358 | Error::BuildTlsOptions { source, .. }
1359 | Error::AllocateRegions { source, .. }
1360 | Error::DeallocateRegions { source, .. }
1361 | Error::BuildCreateRequest { source, .. }
1362 | Error::AllocateRegionRoutes { source, .. }
1363 | Error::AllocateWalOptions { source, .. } => source.retry_hint(),
1364
1365 Error::Other { source, .. }
1366 | Error::ListCatalogs { source, .. }
1367 | Error::ListSchemas { source, .. }
1368 | Error::ListTables { source, .. }
1369 | Error::DowngradeLeader { source, .. } => source.retry_hint(),
1370
1371 Error::SubmitProcedure { source, .. }
1372 | Error::WaitProcedure { source, .. }
1373 | Error::QueryProcedure { source, .. }
1374 | Error::StartProcedureManager { source, .. }
1375 | Error::StopProcedureManager { source, .. }
1376 | Error::RegisterProcedureLoader { source, .. }
1377 | Error::RepartitionSubprocedureStateReceiver { source, .. } => source.retry_hint(),
1378
1379 Error::ShutdownServer { source, .. } | Error::StartHttp { source, .. } => {
1380 source.retry_hint()
1381 }
1382 Error::StartTelemetryTask { source, .. } => source.retry_hint(),
1383 Error::CreateChannel { source, .. } => source.retry_hint(),
1384 Error::RepartitionCreateSubtasks { source, .. } => source.retry_hint(),
1385 Error::SerializePartitionExpr { source, .. }
1386 | Error::DeserializePartitionExpr { source, .. } => source.retry_hint(),
1387
1388 Error::DeleteRecords { error, .. }
1389 | Error::BuildPartitionClient { error, .. }
1390 | Error::GetOffset { error, .. } => rskafka_client_error_to_retry_hint(error),
1391
1392 Error::PusherNotFound { .. }
1393 | Error::PushMessage { .. }
1394 | Error::ExceededDeadline { .. } => RetryHint::NonRetryable,
1395
1396 _ => RetryHint::NonRetryable,
1397 }
1398 }
1399}
1400
1401pub(crate) fn match_for_io_error(err_status: &tonic::Status) -> Option<&std::io::Error> {
1403 let mut err: &(dyn std::error::Error + 'static) = err_status;
1404
1405 loop {
1406 if let Some(io_err) = err.downcast_ref::<std::io::Error>() {
1407 return Some(io_err);
1408 }
1409
1410 if let Some(h2_err) = err.downcast_ref::<h2::Error>()
1413 && let Some(io_err) = h2_err.get_io()
1414 {
1415 return Some(io_err);
1416 }
1417
1418 err = err.source()?;
1419 }
1420}
1421
1422#[cfg(test)]
1423mod tests {
1424 use std::time::Duration;
1425
1426 use common_error::ext::ErrorExt;
1427 use common_error::mock::MockError;
1428 use common_error::status_code::StatusCode;
1429 use rskafka::BackoffError;
1430 use rskafka::client::error::Error as KafkaClientError;
1431 use snafu::ResultExt;
1432
1433 use super::{
1434 BuildPartitionClientSnafu, DeallocateRegionsSnafu, DeleteRecordsSnafu, GetOffsetSnafu,
1435 };
1436
1437 fn retry_failed_kafka_error() -> KafkaClientError {
1438 KafkaClientError::RetryFailed(BackoffError::DeadlineExceded {
1439 deadline: Duration::from_secs(1),
1440 source: Box::new(std::io::Error::other("retry failed")),
1441 })
1442 }
1443
1444 #[test]
1445 fn test_deallocate_regions_is_retryable_when_source_is_retry_later() {
1446 let source = common_meta::error::Error::retry_later(MockError::new(StatusCode::Internal));
1447 let err = Err::<(), _>(source)
1448 .context(DeallocateRegionsSnafu { table_id: 1024_u32 })
1449 .unwrap_err();
1450
1451 assert!(err.is_retryable());
1452 assert!(err.retry_hint().is_retryable());
1453 }
1454
1455 #[test]
1456 fn test_deallocate_regions_is_not_retryable_when_source_is_not_retry_later() {
1457 let source = common_meta::error::UnexpectedSnafu {
1458 err_msg: "mock error",
1459 }
1460 .build();
1461 let err = Err::<(), _>(source)
1462 .context(DeallocateRegionsSnafu { table_id: 1024_u32 })
1463 .unwrap_err();
1464
1465 assert!(!err.is_retryable());
1466 assert!(!err.retry_hint().is_retryable());
1467 }
1468
1469 #[test]
1470 fn test_kafka_retry_failed_errors_are_retryable() {
1471 let delete_records_err = Err::<(), _>(retry_failed_kafka_error())
1472 .context(DeleteRecordsSnafu {
1473 topic: "test_topic",
1474 partition: 0,
1475 offset: 1024u64,
1476 })
1477 .unwrap_err();
1478 let build_partition_client_err = Err::<(), _>(retry_failed_kafka_error())
1479 .context(BuildPartitionClientSnafu {
1480 topic: "test_topic",
1481 partition: 0,
1482 })
1483 .unwrap_err();
1484 let get_offset_err = Err::<(), _>(retry_failed_kafka_error())
1485 .context(GetOffsetSnafu {
1486 topic: "test_topic",
1487 })
1488 .unwrap_err();
1489
1490 assert!(delete_records_err.is_retryable());
1491 assert!(build_partition_client_err.is_retryable());
1492 assert!(get_offset_err.is_retryable());
1493 assert!(delete_records_err.retry_hint().is_retryable());
1494 assert!(build_partition_client_err.retry_hint().is_retryable());
1495 assert!(get_offset_err.retry_hint().is_retryable());
1496 }
1497
1498 #[test]
1499 fn test_kafka_non_retry_failed_errors_are_not_retryable() {
1500 let delete_records_err = Err::<(), _>(KafkaClientError::InvalidResponse("invalid".into()))
1501 .context(DeleteRecordsSnafu {
1502 topic: "test_topic",
1503 partition: 0,
1504 offset: 1024u64,
1505 })
1506 .unwrap_err();
1507 let build_partition_client_err =
1508 Err::<(), _>(KafkaClientError::InvalidResponse("invalid".into()))
1509 .context(BuildPartitionClientSnafu {
1510 topic: "test_topic",
1511 partition: 0,
1512 })
1513 .unwrap_err();
1514 let get_offset_err = Err::<(), _>(KafkaClientError::InvalidResponse("invalid".into()))
1515 .context(GetOffsetSnafu {
1516 topic: "test_topic",
1517 })
1518 .unwrap_err();
1519
1520 assert!(!delete_records_err.is_retryable());
1521 assert!(!build_partition_client_err.is_retryable());
1522 assert!(!get_offset_err.is_retryable());
1523 assert!(!delete_records_err.retry_hint().is_retryable());
1524 assert!(!build_partition_client_err.retry_hint().is_retryable());
1525 assert!(!get_offset_err.retry_hint().is_retryable());
1526 }
1527}