Skip to main content

meta_srv/
error.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use common_error::define_into_tonic_status;
16use common_error::ext::{BoxedError, ErrorExt, RetryHint};
17use common_error::status_code::StatusCode;
18use common_macro::stack_trace_debug;
19use common_meta::DatanodeId;
20use common_procedure::ProcedureId;
21use common_runtime::JoinError;
22use common_wal::kafka::rskafka_client_error_to_retry_hint;
23use snafu::{Location, Snafu};
24use store_api::storage::RegionId;
25use table::metadata::TableId;
26use tokio::sync::mpsc::error::SendError;
27use tonic::codegen::http;
28use uuid::Uuid;
29
30use crate::metasrv::SelectTarget;
31use crate::pubsub::Message;
32use crate::service::mailbox::Channel;
33
34#[derive(Snafu)]
35#[snafu(visibility(pub))]
36#[stack_trace_debug]
37pub enum Error {
38    #[snafu(display("Failed to choose items"))]
39    ChooseItems {
40        #[snafu(implicit)]
41        location: Location,
42        #[snafu(source)]
43        error: rand::distr::weighted::Error,
44    },
45
46    #[snafu(display("Exceeded deadline, operation: {}", operation))]
47    ExceededDeadline {
48        #[snafu(implicit)]
49        location: Location,
50        operation: String,
51    },
52
53    #[snafu(display("The target peer is unavailable temporally: {}", peer_id))]
54    PeerUnavailable {
55        #[snafu(implicit)]
56        location: Location,
57        peer_id: u64,
58    },
59
60    #[snafu(display("Failed to list active frontends"))]
61    ListActiveFrontends {
62        #[snafu(implicit)]
63        location: Location,
64        source: common_meta::error::Error,
65    },
66
67    #[snafu(display("Failed to list active datanodes"))]
68    ListActiveDatanodes {
69        #[snafu(implicit)]
70        location: Location,
71        source: common_meta::error::Error,
72    },
73
74    #[snafu(display("Failed to list active flownodes"))]
75    ListActiveFlownodes {
76        #[snafu(implicit)]
77        location: Location,
78        source: common_meta::error::Error,
79    },
80
81    #[snafu(display("No available frontend"))]
82    NoAvailableFrontend {
83        #[snafu(implicit)]
84        location: Location,
85    },
86
87    #[snafu(display("Another migration procedure is running for region: {}", region_id))]
88    MigrationRunning {
89        #[snafu(implicit)]
90        location: Location,
91        region_id: RegionId,
92    },
93
94    #[snafu(display(
95        "The region migration procedure is completed for region: {}, target_peer: {}",
96        region_id,
97        target_peer_id
98    ))]
99    RegionMigrated {
100        #[snafu(implicit)]
101        location: Location,
102        region_id: RegionId,
103        target_peer_id: u64,
104    },
105
106    #[snafu(display("The region migration procedure aborted, reason: {}", reason))]
107    MigrationAbort {
108        #[snafu(implicit)]
109        location: Location,
110        reason: String,
111    },
112
113    #[snafu(display(
114        "Another procedure is operating the region: {} on peer: {}",
115        region_id,
116        peer_id
117    ))]
118    RegionOperatingRace {
119        #[snafu(implicit)]
120        location: Location,
121        peer_id: DatanodeId,
122        region_id: RegionId,
123    },
124
125    #[snafu(display("Failed to init ddl manager"))]
126    InitDdlManager {
127        #[snafu(implicit)]
128        location: Location,
129        source: common_meta::error::Error,
130    },
131
132    #[snafu(display("Failed to init reconciliation manager"))]
133    InitReconciliationManager {
134        #[snafu(implicit)]
135        location: Location,
136        source: common_meta::error::Error,
137    },
138
139    #[snafu(display("Failed to create default catalog and schema"))]
140    InitMetadata {
141        #[snafu(implicit)]
142        location: Location,
143        source: common_meta::error::Error,
144    },
145
146    #[snafu(display("Failed to allocate next sequence number"))]
147    NextSequence {
148        #[snafu(implicit)]
149        location: Location,
150        source: common_meta::error::Error,
151    },
152
153    #[snafu(display("Failed to set next sequence number"))]
154    SetNextSequence {
155        #[snafu(implicit)]
156        location: Location,
157        source: common_meta::error::Error,
158    },
159
160    #[snafu(display("Failed to peek sequence number"))]
161    PeekSequence {
162        #[snafu(implicit)]
163        location: Location,
164        source: common_meta::error::Error,
165    },
166
167    #[snafu(display("Failed to start telemetry task"))]
168    StartTelemetryTask {
169        #[snafu(implicit)]
170        location: Location,
171        source: common_runtime::error::Error,
172    },
173
174    #[snafu(display("Failed to submit ddl task"))]
175    SubmitDdlTask {
176        #[snafu(implicit)]
177        location: Location,
178        source: common_meta::error::Error,
179    },
180
181    #[snafu(display("Failed to submit reconcile procedure"))]
182    SubmitReconcileProcedure {
183        #[snafu(implicit)]
184        location: Location,
185        source: common_meta::error::Error,
186    },
187
188    #[snafu(display("Failed to invalidate table cache"))]
189    InvalidateTableCache {
190        #[snafu(implicit)]
191        location: Location,
192        source: common_meta::error::Error,
193    },
194
195    #[snafu(display("Failed to list catalogs"))]
196    ListCatalogs {
197        #[snafu(implicit)]
198        location: Location,
199        source: BoxedError,
200    },
201
202    #[snafu(display("Failed to list {}'s schemas", catalog))]
203    ListSchemas {
204        #[snafu(implicit)]
205        location: Location,
206        catalog: String,
207        source: BoxedError,
208    },
209
210    #[snafu(display("Failed to list {}.{}'s tables", catalog, schema))]
211    ListTables {
212        #[snafu(implicit)]
213        location: Location,
214        catalog: String,
215        schema: String,
216        source: BoxedError,
217    },
218
219    #[snafu(display("Failed to join a future"))]
220    Join {
221        #[snafu(implicit)]
222        location: Location,
223        #[snafu(source)]
224        error: JoinError,
225    },
226
227    #[snafu(display(
228        "Failed to request {}, required: {}, but only {} available",
229        select_target,
230        required,
231        available
232    ))]
233    NoEnoughAvailableNode {
234        #[snafu(implicit)]
235        location: Location,
236        required: usize,
237        available: usize,
238        select_target: SelectTarget,
239    },
240
241    #[snafu(display("Failed to send shutdown signal"))]
242    SendShutdownSignal {
243        #[snafu(source)]
244        error: SendError<()>,
245    },
246
247    #[snafu(display("Failed to shutdown {} server", server))]
248    ShutdownServer {
249        #[snafu(implicit)]
250        location: Location,
251        source: servers::error::Error,
252        server: String,
253    },
254
255    #[snafu(display("Empty key is not allowed"))]
256    EmptyKey {
257        #[snafu(implicit)]
258        location: Location,
259    },
260
261    #[snafu(display("Failed to execute via Etcd"))]
262    EtcdFailed {
263        #[snafu(source)]
264        error: etcd_client::Error,
265        #[snafu(implicit)]
266        location: Location,
267    },
268
269    #[snafu(display("Failed to connect to Etcd"))]
270    ConnectEtcd {
271        #[snafu(source)]
272        error: etcd_client::Error,
273        #[snafu(implicit)]
274        location: Location,
275    },
276
277    #[snafu(display("Failed to read file: {}", path))]
278    FileIo {
279        #[snafu(source)]
280        error: std::io::Error,
281        #[snafu(implicit)]
282        location: Location,
283        path: String,
284    },
285
286    #[snafu(display("Failed to bind address {}", addr))]
287    TcpBind {
288        addr: String,
289        #[snafu(source)]
290        error: std::io::Error,
291        #[snafu(implicit)]
292        location: Location,
293    },
294
295    #[snafu(display("Failed to start gRPC server"))]
296    StartGrpc {
297        #[snafu(source)]
298        error: tonic::transport::Error,
299        #[snafu(implicit)]
300        location: Location,
301    },
302
303    #[snafu(display("Failed to start http server"))]
304    StartHttp {
305        #[snafu(implicit)]
306        location: Location,
307        source: servers::error::Error,
308    },
309
310    #[snafu(display("Failed to parse address {}", addr))]
311    ParseAddr {
312        addr: String,
313        #[snafu(source)]
314        error: std::net::AddrParseError,
315    },
316
317    #[snafu(display("Invalid lease key: {}", key))]
318    InvalidLeaseKey {
319        key: String,
320        #[snafu(implicit)]
321        location: Location,
322    },
323
324    #[snafu(display("Invalid datanode stat key: {}", key))]
325    InvalidStatKey {
326        key: String,
327        #[snafu(implicit)]
328        location: Location,
329    },
330
331    #[snafu(display("Invalid inactive region key: {}", key))]
332    InvalidInactiveRegionKey {
333        key: String,
334        #[snafu(implicit)]
335        location: Location,
336    },
337
338    #[snafu(display("Failed to parse lease key from utf8"))]
339    LeaseKeyFromUtf8 {
340        #[snafu(source)]
341        error: std::string::FromUtf8Error,
342        #[snafu(implicit)]
343        location: Location,
344    },
345
346    #[snafu(display("Failed to parse lease value from utf8"))]
347    LeaseValueFromUtf8 {
348        #[snafu(source)]
349        error: std::string::FromUtf8Error,
350        #[snafu(implicit)]
351        location: Location,
352    },
353
354    #[snafu(display("Failed to parse invalid region key from utf8"))]
355    InvalidRegionKeyFromUtf8 {
356        #[snafu(source)]
357        error: std::string::FromUtf8Error,
358        #[snafu(implicit)]
359        location: Location,
360    },
361
362    #[snafu(display("Failed to serialize to json: {}", input))]
363    SerializeToJson {
364        input: String,
365        #[snafu(source)]
366        error: serde_json::error::Error,
367        #[snafu(implicit)]
368        location: Location,
369    },
370
371    #[snafu(display("Failed to deserialize from json: {}", input))]
372    DeserializeFromJson {
373        input: String,
374        #[snafu(source)]
375        error: serde_json::error::Error,
376        #[snafu(implicit)]
377        location: Location,
378    },
379
380    #[snafu(display("Failed to serialize config"))]
381    SerializeConfig {
382        #[snafu(source)]
383        error: serde_json::error::Error,
384        #[snafu(implicit)]
385        location: Location,
386    },
387
388    #[snafu(display("Failed to parse number: {}", err_msg))]
389    ParseNum {
390        err_msg: String,
391        #[snafu(source)]
392        error: std::num::ParseIntError,
393        #[snafu(implicit)]
394        location: Location,
395    },
396
397    #[snafu(display("Failed to parse bool: {}", err_msg))]
398    ParseBool {
399        err_msg: String,
400        #[snafu(source)]
401        error: std::str::ParseBoolError,
402        #[snafu(implicit)]
403        location: Location,
404    },
405
406    #[snafu(display("Failed to downgrade region leader, region: {}", region_id))]
407    DowngradeLeader {
408        region_id: RegionId,
409        #[snafu(implicit)]
410        location: Location,
411        #[snafu(source)]
412        source: BoxedError,
413    },
414
415    #[snafu(display("Region's leader peer changed: {}", msg))]
416    LeaderPeerChanged {
417        msg: String,
418        #[snafu(implicit)]
419        location: Location,
420    },
421
422    #[snafu(display("Invalid arguments: {}", err_msg))]
423    InvalidArguments {
424        err_msg: String,
425        #[snafu(implicit)]
426        location: Location,
427    },
428
429    #[snafu(display("Manual GC is rejected because maintenance mode is enabled"))]
430    ManualGcRejectedByMaintenanceMode {
431        #[snafu(implicit)]
432        location: Location,
433    },
434
435    #[cfg(feature = "mysql_kvbackend")]
436    #[snafu(display("Failed to parse mysql url: {}", mysql_url))]
437    ParseMySqlUrl {
438        #[snafu(source)]
439        error: sqlx::error::Error,
440        mysql_url: String,
441        #[snafu(implicit)]
442        location: Location,
443    },
444
445    #[cfg(feature = "mysql_kvbackend")]
446    #[snafu(display("Failed to decode sql value"))]
447    DecodeSqlValue {
448        #[snafu(source)]
449        error: sqlx::error::Error,
450        #[snafu(implicit)]
451        location: Location,
452    },
453
454    #[snafu(display("Failed to find table route for {table_id}"))]
455    TableRouteNotFound {
456        table_id: TableId,
457        #[snafu(implicit)]
458        location: Location,
459    },
460
461    #[snafu(display("Failed to find table route for {region_id}"))]
462    RegionRouteNotFound {
463        region_id: RegionId,
464        #[snafu(implicit)]
465        location: Location,
466    },
467
468    #[snafu(display("Table info not found: {}", table_id))]
469    TableInfoNotFound {
470        table_id: TableId,
471        #[snafu(implicit)]
472        location: Location,
473    },
474
475    #[snafu(display("Datanode table not found: {}, datanode: {}", table_id, datanode_id))]
476    DatanodeTableNotFound {
477        table_id: TableId,
478        datanode_id: DatanodeId,
479        #[snafu(implicit)]
480        location: Location,
481    },
482
483    #[snafu(display("Metasrv has no leader at this moment"))]
484    NoLeader {
485        #[snafu(implicit)]
486        location: Location,
487    },
488
489    #[snafu(display("Leader lease expired"))]
490    LeaderLeaseExpired {
491        #[snafu(implicit)]
492        location: Location,
493    },
494
495    #[snafu(display("Leader lease changed during election"))]
496    LeaderLeaseChanged {
497        #[snafu(implicit)]
498        location: Location,
499    },
500
501    #[snafu(display("Table {} not found", name))]
502    TableNotFound {
503        name: String,
504        #[snafu(implicit)]
505        location: Location,
506    },
507
508    #[snafu(display("Unsupported selector type, {}", selector_type))]
509    UnsupportedSelectorType {
510        selector_type: String,
511        #[snafu(implicit)]
512        location: Location,
513    },
514
515    #[snafu(display("Unexpected, violated: {violated}"))]
516    Unexpected {
517        violated: String,
518        #[snafu(implicit)]
519        location: Location,
520    },
521
522    #[snafu(display("Failed to create gRPC channel"))]
523    CreateChannel {
524        #[snafu(implicit)]
525        location: Location,
526        source: common_grpc::error::Error,
527    },
528
529    #[snafu(display("Failed to batch get KVs from leader's in_memory kv store"))]
530    BatchGet {
531        #[snafu(source)]
532        error: tonic::Status,
533        #[snafu(implicit)]
534        location: Location,
535    },
536
537    #[snafu(display("Failed to batch range KVs from leader's in_memory kv store"))]
538    Range {
539        #[snafu(source)]
540        error: tonic::Status,
541        #[snafu(implicit)]
542        location: Location,
543    },
544
545    #[snafu(display("Response header not found"))]
546    ResponseHeaderNotFound {
547        #[snafu(implicit)]
548        location: Location,
549    },
550
551    #[snafu(display("The requested meta node is not leader, node addr: {}", node_addr))]
552    IsNotLeader {
553        node_addr: String,
554        #[snafu(implicit)]
555        location: Location,
556    },
557
558    #[snafu(display("Invalid http body"))]
559    InvalidHttpBody {
560        #[snafu(source)]
561        error: http::Error,
562        #[snafu(implicit)]
563        location: Location,
564    },
565
566    #[snafu(display(
567        "The number of retries for the grpc call {} exceeded the limit, {}",
568        func_name,
569        retry_num
570    ))]
571    ExceededRetryLimit {
572        func_name: String,
573        retry_num: usize,
574        #[snafu(implicit)]
575        location: Location,
576    },
577
578    #[snafu(display("Invalid utf-8 value"))]
579    InvalidUtf8Value {
580        #[snafu(source)]
581        error: std::string::FromUtf8Error,
582        #[snafu(implicit)]
583        location: Location,
584    },
585
586    #[snafu(display("Missing required parameter, param: {:?}", param))]
587    MissingRequiredParameter { param: String },
588
589    #[snafu(display("Failed to start procedure manager"))]
590    StartProcedureManager {
591        #[snafu(implicit)]
592        location: Location,
593        source: common_procedure::Error,
594    },
595
596    #[snafu(display("Failed to stop procedure manager"))]
597    StopProcedureManager {
598        #[snafu(implicit)]
599        location: Location,
600        source: common_procedure::Error,
601    },
602
603    #[snafu(display("Failed to wait procedure done"))]
604    WaitProcedure {
605        #[snafu(implicit)]
606        location: Location,
607        source: common_procedure::Error,
608    },
609
610    #[snafu(display("Failed to query procedure state"))]
611    QueryProcedure {
612        #[snafu(implicit)]
613        location: Location,
614        source: common_procedure::Error,
615    },
616
617    #[snafu(display("Procedure not found: {pid}"))]
618    ProcedureNotFound {
619        #[snafu(implicit)]
620        location: Location,
621        pid: String,
622    },
623
624    #[snafu(display("Failed to submit procedure"))]
625    SubmitProcedure {
626        #[snafu(implicit)]
627        location: Location,
628        source: common_procedure::Error,
629    },
630
631    #[snafu(display("A prune task for topic {} is already running", topic))]
632    PruneTaskAlreadyRunning {
633        topic: String,
634        #[snafu(implicit)]
635        location: Location,
636    },
637
638    #[snafu(display("Schema already exists, name: {schema_name}"))]
639    SchemaAlreadyExists {
640        schema_name: String,
641        #[snafu(implicit)]
642        location: Location,
643    },
644
645    #[snafu(display("Table already exists: {table_name}"))]
646    TableAlreadyExists {
647        table_name: String,
648        #[snafu(implicit)]
649        location: Location,
650    },
651
652    #[snafu(display("Pusher not found: {pusher_id}"))]
653    PusherNotFound {
654        pusher_id: String,
655        #[snafu(implicit)]
656        location: Location,
657    },
658
659    #[snafu(display("Failed to push message: {err_msg}"))]
660    PushMessage {
661        err_msg: String,
662        #[snafu(implicit)]
663        location: Location,
664    },
665
666    #[snafu(display("Mailbox already closed: {id}"))]
667    MailboxClosed {
668        id: u64,
669        #[snafu(implicit)]
670        location: Location,
671    },
672
673    #[snafu(display("Mailbox timeout: {id}"))]
674    MailboxTimeout {
675        id: u64,
676        #[snafu(implicit)]
677        location: Location,
678    },
679
680    #[snafu(display("Mailbox receiver got an error: {id}, {err_msg}"))]
681    MailboxReceiver {
682        id: u64,
683        err_msg: String,
684        #[snafu(implicit)]
685        location: Location,
686    },
687
688    #[snafu(display("Mailbox channel closed: {channel}"))]
689    MailboxChannelClosed {
690        channel: Channel,
691        #[snafu(implicit)]
692        location: Location,
693    },
694
695    #[snafu(display("Missing request header"))]
696    MissingRequestHeader {
697        #[snafu(implicit)]
698        location: Location,
699    },
700
701    #[snafu(display("Failed to register procedure loader, type name: {}", type_name))]
702    RegisterProcedureLoader {
703        type_name: String,
704        #[snafu(implicit)]
705        location: Location,
706        source: common_procedure::error::Error,
707    },
708
709    #[snafu(display(
710        "Received unexpected instruction reply, mailbox message: {}, reason: {}",
711        mailbox_message,
712        reason
713    ))]
714    UnexpectedInstructionReply {
715        mailbox_message: String,
716        reason: String,
717        #[snafu(implicit)]
718        location: Location,
719    },
720
721    #[snafu(display("Expected to retry later, reason: {}", reason))]
722    RetryLater {
723        reason: String,
724        #[snafu(implicit)]
725        location: Location,
726    },
727
728    #[snafu(display("Expected to retry later, reason: {}", reason))]
729    RetryLaterWithSource {
730        reason: String,
731        #[snafu(implicit)]
732        location: Location,
733        source: BoxedError,
734    },
735
736    #[snafu(display("Failed to convert proto data"))]
737    ConvertProtoData {
738        #[snafu(implicit)]
739        location: Location,
740        source: common_meta::error::Error,
741    },
742
743    // this error is used for custom error mapping
744    // please do not delete it
745    #[snafu(display("Other error"))]
746    Other {
747        source: BoxedError,
748        #[snafu(implicit)]
749        location: Location,
750    },
751
752    #[snafu(display("Table metadata manager error"))]
753    TableMetadataManager {
754        source: common_meta::error::Error,
755        #[snafu(implicit)]
756        location: Location,
757    },
758
759    #[snafu(display("Runtime switch manager error"))]
760    RuntimeSwitchManager {
761        source: common_meta::error::Error,
762        #[snafu(implicit)]
763        location: Location,
764    },
765
766    #[snafu(display("Keyvalue backend error"))]
767    KvBackend {
768        source: common_meta::error::Error,
769        #[snafu(implicit)]
770        location: Location,
771    },
772
773    #[snafu(display("Failed to publish message"))]
774    PublishMessage {
775        #[snafu(source)]
776        error: SendError<Message>,
777        #[snafu(implicit)]
778        location: Location,
779    },
780
781    #[snafu(display("Too many partitions"))]
782    TooManyPartitions {
783        #[snafu(implicit)]
784        location: Location,
785    },
786
787    #[snafu(display("Failed to create repartition subtasks"))]
788    RepartitionCreateSubtasks {
789        source: partition::error::Error,
790        #[snafu(implicit)]
791        location: Location,
792    },
793
794    #[snafu(display(
795        "Source partition expression '{}' does not match any existing region",
796        expr
797    ))]
798    RepartitionSourceExprMismatch {
799        expr: String,
800        #[snafu(implicit)]
801        location: Location,
802    },
803
804    #[snafu(display(
805        "Failed to get the state receiver for repartition subprocedure {}",
806        procedure_id
807    ))]
808    RepartitionSubprocedureStateReceiver {
809        procedure_id: ProcedureId,
810        #[snafu(source)]
811        source: common_procedure::Error,
812        #[snafu(implicit)]
813        location: Location,
814    },
815
816    #[snafu(display("Unsupported operation {}", operation))]
817    Unsupported {
818        operation: String,
819        #[snafu(implicit)]
820        location: Location,
821    },
822
823    #[snafu(display("Unexpected table route type: {}", err_msg))]
824    UnexpectedLogicalRouteTable {
825        #[snafu(implicit)]
826        location: Location,
827        err_msg: String,
828        source: common_meta::error::Error,
829    },
830
831    #[snafu(display("Failed to save cluster info"))]
832    SaveClusterInfo {
833        #[snafu(implicit)]
834        location: Location,
835        source: common_meta::error::Error,
836    },
837
838    #[snafu(display("Invalid cluster info format"))]
839    InvalidClusterInfoFormat {
840        #[snafu(implicit)]
841        location: Location,
842        source: common_meta::error::Error,
843    },
844
845    #[snafu(display("Invalid datanode stat format"))]
846    InvalidDatanodeStatFormat {
847        #[snafu(implicit)]
848        location: Location,
849        source: common_meta::error::Error,
850    },
851
852    #[snafu(display("Invalid node info format"))]
853    InvalidNodeInfoFormat {
854        #[snafu(implicit)]
855        location: Location,
856        source: common_meta::error::Error,
857    },
858
859    #[snafu(display("Failed to serialize options to TOML"))]
860    TomlFormat {
861        #[snafu(implicit)]
862        location: Location,
863        #[snafu(source(from(common_config::error::Error, Box::new)))]
864        source: Box<common_config::error::Error>,
865    },
866
867    #[cfg(feature = "pg_kvbackend")]
868    #[snafu(display("Failed to execute via postgres, sql: {}", sql))]
869    PostgresExecution {
870        #[snafu(source)]
871        error: tokio_postgres::Error,
872        sql: String,
873        #[snafu(implicit)]
874        location: Location,
875    },
876
877    #[cfg(feature = "pg_kvbackend")]
878    #[snafu(display("Failed to get Postgres client"))]
879    GetPostgresClient {
880        #[snafu(implicit)]
881        location: Location,
882        #[snafu(source)]
883        error: deadpool::managed::PoolError<tokio_postgres::Error>,
884    },
885
886    #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
887    #[snafu(display("Sql execution timeout, sql: {}, duration: {:?}", sql, duration))]
888    SqlExecutionTimeout {
889        #[snafu(implicit)]
890        location: Location,
891        sql: String,
892        duration: std::time::Duration,
893    },
894
895    #[cfg(feature = "pg_kvbackend")]
896    #[snafu(display("Failed to create connection pool for Postgres"))]
897    CreatePostgresPool {
898        #[snafu(source)]
899        error: deadpool_postgres::CreatePoolError,
900        #[snafu(implicit)]
901        location: Location,
902    },
903
904    #[cfg(feature = "pg_kvbackend")]
905    #[snafu(display("Failed to get connection from Postgres pool: {}", reason))]
906    GetPostgresConnection {
907        reason: String,
908        #[snafu(implicit)]
909        location: Location,
910    },
911
912    #[cfg(feature = "mysql_kvbackend")]
913    #[snafu(display("Failed to execute via mysql, sql: {}", sql))]
914    MySqlExecution {
915        #[snafu(source)]
916        error: sqlx::Error,
917        #[snafu(implicit)]
918        location: Location,
919        sql: String,
920    },
921
922    #[cfg(feature = "mysql_kvbackend")]
923    #[snafu(display("Failed to create mysql pool"))]
924    CreateMySqlPool {
925        #[snafu(source)]
926        error: sqlx::Error,
927        #[snafu(implicit)]
928        location: Location,
929    },
930
931    #[cfg(feature = "mysql_kvbackend")]
932    #[snafu(display("Failed to acquire mysql client from pool"))]
933    AcquireMySqlClient {
934        #[snafu(source)]
935        error: sqlx::Error,
936        #[snafu(implicit)]
937        location: Location,
938    },
939
940    #[snafu(display("Handler not found: {}", name))]
941    HandlerNotFound {
942        name: String,
943        #[snafu(implicit)]
944        location: Location,
945    },
946
947    #[snafu(display("Flow state handler error"))]
948    FlowStateHandler {
949        #[snafu(implicit)]
950        location: Location,
951        source: common_meta::error::Error,
952    },
953
954    #[snafu(display("Failed to build wal provider"))]
955    BuildWalProvider {
956        #[snafu(implicit)]
957        location: Location,
958        source: common_meta::error::Error,
959    },
960
961    #[snafu(display("Failed to build kafka client."))]
962    BuildKafkaClient {
963        #[snafu(implicit)]
964        location: Location,
965        #[snafu(source)]
966        error: common_meta::error::Error,
967    },
968
969    #[snafu(display(
970        "Failed to build a Kafka partition client, topic: {}, partition: {}",
971        topic,
972        partition
973    ))]
974    BuildPartitionClient {
975        topic: String,
976        partition: i32,
977        #[snafu(implicit)]
978        location: Location,
979        #[snafu(source)]
980        error: rskafka::client::error::Error,
981    },
982
983    #[snafu(display(
984        "Failed to delete records from Kafka, topic: {}, partition: {}, offset: {}",
985        topic,
986        partition,
987        offset
988    ))]
989    DeleteRecords {
990        #[snafu(implicit)]
991        location: Location,
992        #[snafu(source)]
993        error: rskafka::client::error::Error,
994        topic: String,
995        partition: i32,
996        offset: u64,
997    },
998
999    #[snafu(display("Failed to get offset from Kafka, topic: {}", topic))]
1000    GetOffset {
1001        topic: String,
1002        #[snafu(implicit)]
1003        location: Location,
1004        #[snafu(source)]
1005        error: rskafka::client::error::Error,
1006    },
1007
1008    #[snafu(display("Failed to update the TopicNameValue in kvbackend, topic: {}", topic))]
1009    UpdateTopicNameValue {
1010        topic: String,
1011        #[snafu(implicit)]
1012        location: Location,
1013        #[snafu(source)]
1014        source: common_meta::error::Error,
1015    },
1016
1017    #[snafu(display("Failed to build tls options"))]
1018    BuildTlsOptions {
1019        #[snafu(implicit)]
1020        location: Location,
1021        #[snafu(source)]
1022        source: common_meta::error::Error,
1023    },
1024
1025    #[snafu(display(
1026        "Repartition group {} source region missing, region id: {}",
1027        group_id,
1028        region_id
1029    ))]
1030    RepartitionSourceRegionMissing {
1031        group_id: Uuid,
1032        region_id: RegionId,
1033        #[snafu(implicit)]
1034        location: Location,
1035    },
1036
1037    #[snafu(display(
1038        "Repartition group {} target region missing, region id: {}",
1039        group_id,
1040        region_id
1041    ))]
1042    RepartitionTargetRegionMissing {
1043        group_id: Uuid,
1044        region_id: RegionId,
1045        #[snafu(implicit)]
1046        location: Location,
1047    },
1048
1049    #[snafu(display("Failed to serialize partition expression"))]
1050    SerializePartitionExpr {
1051        #[snafu(source)]
1052        source: partition::error::Error,
1053        #[snafu(implicit)]
1054        location: Location,
1055    },
1056
1057    #[snafu(display("Failed to deserialize partition expression"))]
1058    DeserializePartitionExpr {
1059        #[snafu(source)]
1060        source: partition::error::Error,
1061        #[snafu(implicit)]
1062        location: Location,
1063    },
1064
1065    #[snafu(display("Empty partition expression"))]
1066    EmptyPartitionExpr {
1067        #[snafu(implicit)]
1068        location: Location,
1069    },
1070
1071    #[snafu(display(
1072        "Partition expression mismatch, region id: {}, expected: {}, actual: {}",
1073        region_id,
1074        expected,
1075        actual
1076    ))]
1077    PartitionExprMismatch {
1078        region_id: RegionId,
1079        expected: String,
1080        actual: String,
1081        #[snafu(implicit)]
1082        location: Location,
1083    },
1084
1085    #[snafu(display("Failed to allocate regions for table: {}", table_id))]
1086    AllocateRegions {
1087        #[snafu(implicit)]
1088        location: Location,
1089        table_id: TableId,
1090        #[snafu(source)]
1091        source: common_meta::error::Error,
1092    },
1093
1094    #[snafu(display("Failed to deallocate regions for table: {}", table_id))]
1095    DeallocateRegions {
1096        #[snafu(implicit)]
1097        location: Location,
1098        table_id: TableId,
1099        #[snafu(source)]
1100        source: common_meta::error::Error,
1101    },
1102
1103    #[snafu(display("Failed to build create request for table: {}", table_id))]
1104    BuildCreateRequest {
1105        #[snafu(implicit)]
1106        location: Location,
1107        table_id: TableId,
1108        #[snafu(source)]
1109        source: common_meta::error::Error,
1110    },
1111
1112    #[snafu(display("Failed to allocate region routes for table: {}", table_id))]
1113    AllocateRegionRoutes {
1114        #[snafu(implicit)]
1115        location: Location,
1116        table_id: TableId,
1117        #[snafu(source)]
1118        source: common_meta::error::Error,
1119    },
1120
1121    #[snafu(display("Failed to allocate wal options for table: {}", table_id))]
1122    AllocateWalOptions {
1123        #[snafu(implicit)]
1124        location: Location,
1125        table_id: TableId,
1126        #[snafu(source)]
1127        source: common_meta::error::Error,
1128    },
1129}
1130
1131impl Error {
1132    /// Returns `true` if the error is retryable.
1133    pub fn is_retryable(&self) -> bool {
1134        self.retry_hint().is_retryable()
1135    }
1136}
1137
1138pub type Result<T> = std::result::Result<T, Error>;
1139
1140define_into_tonic_status!(Error);
1141
1142impl ErrorExt for Error {
1143    fn status_code(&self) -> StatusCode {
1144        match self {
1145            Error::EtcdFailed { .. }
1146            | Error::ConnectEtcd { .. }
1147            | Error::FileIo { .. }
1148            | Error::TcpBind { .. }
1149            | Error::SerializeConfig { .. }
1150            | Error::SerializeToJson { .. }
1151            | Error::DeserializeFromJson { .. }
1152            | Error::NoLeader { .. }
1153            | Error::LeaderLeaseExpired { .. }
1154            | Error::LeaderLeaseChanged { .. }
1155            | Error::CreateChannel { .. }
1156            | Error::BatchGet { .. }
1157            | Error::Range { .. }
1158            | Error::ResponseHeaderNotFound { .. }
1159            | Error::InvalidHttpBody { .. }
1160            | Error::ExceededRetryLimit { .. }
1161            | Error::SendShutdownSignal { .. }
1162            | Error::PushMessage { .. }
1163            | Error::MailboxClosed { .. }
1164            | Error::MailboxReceiver { .. }
1165            | Error::StartGrpc { .. }
1166            | Error::PublishMessage { .. }
1167            | Error::Join { .. }
1168            | Error::ChooseItems { .. }
1169            | Error::FlowStateHandler { .. }
1170            | Error::BuildWalProvider { .. }
1171            | Error::BuildPartitionClient { .. }
1172            | Error::BuildKafkaClient { .. } => StatusCode::Internal,
1173
1174            Error::DeleteRecords { .. }
1175            | Error::GetOffset { .. }
1176            | Error::PeerUnavailable { .. }
1177            | Error::PusherNotFound { .. } => StatusCode::Unexpected,
1178            Error::MailboxTimeout { .. } | Error::ExceededDeadline { .. } => StatusCode::Cancelled,
1179            Error::PruneTaskAlreadyRunning { .. }
1180            | Error::RetryLater { .. }
1181            | Error::MailboxChannelClosed { .. }
1182            | Error::IsNotLeader { .. } => StatusCode::IllegalState,
1183            Error::RetryLaterWithSource { source, .. } => source.status_code(),
1184            Error::SerializePartitionExpr { source, .. }
1185            | Error::DeserializePartitionExpr { source, .. } => source.status_code(),
1186
1187            Error::Unsupported { .. } => StatusCode::Unsupported,
1188
1189            Error::SchemaAlreadyExists { .. } => StatusCode::DatabaseAlreadyExists,
1190
1191            Error::TableAlreadyExists { .. } => StatusCode::TableAlreadyExists,
1192            Error::EmptyKey { .. }
1193            | Error::MissingRequiredParameter { .. }
1194            | Error::MissingRequestHeader { .. }
1195            | Error::InvalidLeaseKey { .. }
1196            | Error::InvalidStatKey { .. }
1197            | Error::InvalidInactiveRegionKey { .. }
1198            | Error::ParseNum { .. }
1199            | Error::ParseBool { .. }
1200            | Error::ParseAddr { .. }
1201            | Error::UnsupportedSelectorType { .. }
1202            | Error::InvalidArguments { .. }
1203            | Error::ManualGcRejectedByMaintenanceMode { .. }
1204            | Error::ProcedureNotFound { .. }
1205            | Error::TooManyPartitions { .. }
1206            | Error::TomlFormat { .. }
1207            | Error::HandlerNotFound { .. }
1208            | Error::LeaderPeerChanged { .. }
1209            | Error::RepartitionSourceRegionMissing { .. }
1210            | Error::RepartitionTargetRegionMissing { .. }
1211            | Error::PartitionExprMismatch { .. }
1212            | Error::RepartitionSourceExprMismatch { .. }
1213            | Error::EmptyPartitionExpr { .. } => StatusCode::InvalidArguments,
1214            Error::LeaseKeyFromUtf8 { .. }
1215            | Error::LeaseValueFromUtf8 { .. }
1216            | Error::InvalidRegionKeyFromUtf8 { .. }
1217            | Error::TableRouteNotFound { .. }
1218            | Error::TableInfoNotFound { .. }
1219            | Error::DatanodeTableNotFound { .. }
1220            | Error::InvalidUtf8Value { .. }
1221            | Error::UnexpectedInstructionReply { .. }
1222            | Error::Unexpected { .. }
1223            | Error::RegionOperatingRace { .. }
1224            | Error::RegionRouteNotFound { .. }
1225            | Error::MigrationAbort { .. }
1226            | Error::MigrationRunning { .. }
1227            | Error::RegionMigrated { .. } => StatusCode::Unexpected,
1228            Error::TableNotFound { .. } => StatusCode::TableNotFound,
1229            Error::SaveClusterInfo { source, .. }
1230            | Error::InvalidClusterInfoFormat { source, .. }
1231            | Error::InvalidDatanodeStatFormat { source, .. }
1232            | Error::InvalidNodeInfoFormat { source, .. } => source.status_code(),
1233            Error::InvalidateTableCache { source, .. } => source.status_code(),
1234            Error::SubmitProcedure { source, .. }
1235            | Error::WaitProcedure { source, .. }
1236            | Error::QueryProcedure { source, .. } => source.status_code(),
1237            Error::ShutdownServer { source, .. } | Error::StartHttp { source, .. } => {
1238                source.status_code()
1239            }
1240            Error::StartProcedureManager { source, .. }
1241            | Error::StopProcedureManager { source, .. } => source.status_code(),
1242
1243            Error::ListCatalogs { source, .. }
1244            | Error::ListSchemas { source, .. }
1245            | Error::ListTables { source, .. } => source.status_code(),
1246            Error::StartTelemetryTask { source, .. } => source.status_code(),
1247
1248            Error::NextSequence { source, .. }
1249            | Error::SetNextSequence { source, .. }
1250            | Error::PeekSequence { source, .. } => source.status_code(),
1251            Error::DowngradeLeader { source, .. } => source.status_code(),
1252            Error::RegisterProcedureLoader { source, .. } => source.status_code(),
1253            Error::SubmitDdlTask { source, .. }
1254            | Error::SubmitReconcileProcedure { source, .. } => source.status_code(),
1255            Error::ConvertProtoData { source, .. }
1256            | Error::TableMetadataManager { source, .. }
1257            | Error::RuntimeSwitchManager { source, .. }
1258            | Error::KvBackend { source, .. }
1259            | Error::UnexpectedLogicalRouteTable { source, .. }
1260            | Error::UpdateTopicNameValue { source, .. } => source.status_code(),
1261            Error::ListActiveFrontends { source, .. }
1262            | Error::ListActiveDatanodes { source, .. }
1263            | Error::ListActiveFlownodes { source, .. } => source.status_code(),
1264            Error::NoAvailableFrontend { .. } => StatusCode::IllegalState,
1265
1266            Error::InitMetadata { source, .. }
1267            | Error::InitDdlManager { source, .. }
1268            | Error::InitReconciliationManager { source, .. } => source.status_code(),
1269
1270            Error::BuildTlsOptions { source, .. } => source.status_code(),
1271            Error::Other { source, .. } => source.status_code(),
1272            Error::RepartitionCreateSubtasks { source, .. } => source.status_code(),
1273            Error::RepartitionSubprocedureStateReceiver { source, .. } => source.status_code(),
1274            Error::AllocateRegions { source, .. } => source.status_code(),
1275            Error::DeallocateRegions { source, .. } => source.status_code(),
1276            Error::AllocateRegionRoutes { source, .. } => source.status_code(),
1277            Error::AllocateWalOptions { source, .. } => source.status_code(),
1278            Error::BuildCreateRequest { source, .. } => source.status_code(),
1279            Error::NoEnoughAvailableNode { .. } => StatusCode::RuntimeResourcesExhausted,
1280
1281            #[cfg(feature = "pg_kvbackend")]
1282            Error::CreatePostgresPool { .. }
1283            | Error::GetPostgresClient { .. }
1284            | Error::GetPostgresConnection { .. }
1285            | Error::PostgresExecution { .. } => StatusCode::Internal,
1286            #[cfg(feature = "mysql_kvbackend")]
1287            Error::MySqlExecution { .. }
1288            | Error::CreateMySqlPool { .. }
1289            | Error::ParseMySqlUrl { .. }
1290            | Error::DecodeSqlValue { .. }
1291            | Error::AcquireMySqlClient { .. } => StatusCode::Internal,
1292            #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
1293            Error::SqlExecutionTimeout { .. } => StatusCode::Internal,
1294        }
1295    }
1296
1297    fn as_any(&self) -> &dyn std::any::Any {
1298        self
1299    }
1300
1301    fn retry_hint(&self) -> RetryHint {
1302        match self {
1303            Error::RetryLater { .. }
1304            | Error::RetryLaterWithSource { .. }
1305            | Error::MailboxTimeout { .. }
1306            | Error::NoEnoughAvailableNode { .. }
1307            | Error::NoLeader { .. }
1308            | Error::LeaderLeaseExpired { .. }
1309            | Error::LeaderLeaseChanged { .. }
1310            | Error::PeerUnavailable { .. } => RetryHint::Retryable,
1311
1312            Error::ConnectEtcd { error, .. } | Error::EtcdFailed { error, .. } => {
1313                common_meta::error::retry_hint_from_etcd_error(error)
1314            }
1315
1316            #[cfg(feature = "pg_kvbackend")]
1317            Error::PostgresExecution { error, .. } => {
1318                common_meta::error::retry_hint_from_postgres_error(error)
1319            }
1320            #[cfg(feature = "pg_kvbackend")]
1321            Error::GetPostgresClient { error, .. } => {
1322                common_meta::error::retry_hint_from_postgres_pool_error(error)
1323            }
1324            #[cfg(feature = "mysql_kvbackend")]
1325            Error::MySqlExecution { error, .. }
1326            | Error::CreateMySqlPool { error, .. }
1327            | Error::AcquireMySqlClient { error, .. } => {
1328                common_meta::error::retry_hint_from_sqlx_error(error)
1329            }
1330            #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
1331            Error::SqlExecutionTimeout { .. } => RetryHint::Retryable,
1332
1333            Error::ListActiveFrontends { source, .. }
1334            | Error::ListActiveDatanodes { source, .. }
1335            | Error::ListActiveFlownodes { source, .. }
1336            | Error::InitDdlManager { source, .. }
1337            | Error::InitReconciliationManager { source, .. }
1338            | Error::InitMetadata { source, .. }
1339            | Error::NextSequence { source, .. }
1340            | Error::SetNextSequence { source, .. }
1341            | Error::PeekSequence { source, .. }
1342            | Error::SubmitDdlTask { source, .. }
1343            | Error::SubmitReconcileProcedure { source, .. }
1344            | Error::InvalidateTableCache { source, .. }
1345            | Error::ConvertProtoData { source, .. }
1346            | Error::TableMetadataManager { source, .. }
1347            | Error::RuntimeSwitchManager { source, .. }
1348            | Error::KvBackend { source, .. }
1349            | Error::UnexpectedLogicalRouteTable { source, .. }
1350            | Error::SaveClusterInfo { source, .. }
1351            | Error::InvalidClusterInfoFormat { source, .. }
1352            | Error::InvalidDatanodeStatFormat { source, .. }
1353            | Error::InvalidNodeInfoFormat { source, .. }
1354            | Error::FlowStateHandler { source, .. }
1355            | Error::BuildWalProvider { source, .. }
1356            | Error::BuildKafkaClient { error: source, .. }
1357            | Error::UpdateTopicNameValue { source, .. }
1358            | Error::BuildTlsOptions { source, .. }
1359            | Error::AllocateRegions { source, .. }
1360            | Error::DeallocateRegions { source, .. }
1361            | Error::BuildCreateRequest { source, .. }
1362            | Error::AllocateRegionRoutes { source, .. }
1363            | Error::AllocateWalOptions { source, .. } => source.retry_hint(),
1364
1365            Error::Other { source, .. }
1366            | Error::ListCatalogs { source, .. }
1367            | Error::ListSchemas { source, .. }
1368            | Error::ListTables { source, .. }
1369            | Error::DowngradeLeader { source, .. } => source.retry_hint(),
1370
1371            Error::SubmitProcedure { source, .. }
1372            | Error::WaitProcedure { source, .. }
1373            | Error::QueryProcedure { source, .. }
1374            | Error::StartProcedureManager { source, .. }
1375            | Error::StopProcedureManager { source, .. }
1376            | Error::RegisterProcedureLoader { source, .. }
1377            | Error::RepartitionSubprocedureStateReceiver { source, .. } => source.retry_hint(),
1378
1379            Error::ShutdownServer { source, .. } | Error::StartHttp { source, .. } => {
1380                source.retry_hint()
1381            }
1382            Error::StartTelemetryTask { source, .. } => source.retry_hint(),
1383            Error::CreateChannel { source, .. } => source.retry_hint(),
1384            Error::RepartitionCreateSubtasks { source, .. } => source.retry_hint(),
1385            Error::SerializePartitionExpr { source, .. }
1386            | Error::DeserializePartitionExpr { source, .. } => source.retry_hint(),
1387
1388            Error::DeleteRecords { error, .. }
1389            | Error::BuildPartitionClient { error, .. }
1390            | Error::GetOffset { error, .. } => rskafka_client_error_to_retry_hint(error),
1391
1392            Error::PusherNotFound { .. }
1393            | Error::PushMessage { .. }
1394            | Error::ExceededDeadline { .. } => RetryHint::NonRetryable,
1395
1396            _ => RetryHint::NonRetryable,
1397        }
1398    }
1399}
1400
1401// for form tonic
1402pub(crate) fn match_for_io_error(err_status: &tonic::Status) -> Option<&std::io::Error> {
1403    let mut err: &(dyn std::error::Error + 'static) = err_status;
1404
1405    loop {
1406        if let Some(io_err) = err.downcast_ref::<std::io::Error>() {
1407            return Some(io_err);
1408        }
1409
1410        // h2::Error do not expose std::io::Error with `source()`
1411        // https://github.com/hyperium/h2/pull/462
1412        if let Some(h2_err) = err.downcast_ref::<h2::Error>()
1413            && let Some(io_err) = h2_err.get_io()
1414        {
1415            return Some(io_err);
1416        }
1417
1418        err = err.source()?;
1419    }
1420}
1421
1422#[cfg(test)]
1423mod tests {
1424    use std::time::Duration;
1425
1426    use common_error::ext::ErrorExt;
1427    use common_error::mock::MockError;
1428    use common_error::status_code::StatusCode;
1429    use rskafka::BackoffError;
1430    use rskafka::client::error::Error as KafkaClientError;
1431    use snafu::ResultExt;
1432
1433    use super::{
1434        BuildPartitionClientSnafu, DeallocateRegionsSnafu, DeleteRecordsSnafu, GetOffsetSnafu,
1435    };
1436
1437    fn retry_failed_kafka_error() -> KafkaClientError {
1438        KafkaClientError::RetryFailed(BackoffError::DeadlineExceded {
1439            deadline: Duration::from_secs(1),
1440            source: Box::new(std::io::Error::other("retry failed")),
1441        })
1442    }
1443
1444    #[test]
1445    fn test_deallocate_regions_is_retryable_when_source_is_retry_later() {
1446        let source = common_meta::error::Error::retry_later(MockError::new(StatusCode::Internal));
1447        let err = Err::<(), _>(source)
1448            .context(DeallocateRegionsSnafu { table_id: 1024_u32 })
1449            .unwrap_err();
1450
1451        assert!(err.is_retryable());
1452        assert!(err.retry_hint().is_retryable());
1453    }
1454
1455    #[test]
1456    fn test_deallocate_regions_is_not_retryable_when_source_is_not_retry_later() {
1457        let source = common_meta::error::UnexpectedSnafu {
1458            err_msg: "mock error",
1459        }
1460        .build();
1461        let err = Err::<(), _>(source)
1462            .context(DeallocateRegionsSnafu { table_id: 1024_u32 })
1463            .unwrap_err();
1464
1465        assert!(!err.is_retryable());
1466        assert!(!err.retry_hint().is_retryable());
1467    }
1468
1469    #[test]
1470    fn test_kafka_retry_failed_errors_are_retryable() {
1471        let delete_records_err = Err::<(), _>(retry_failed_kafka_error())
1472            .context(DeleteRecordsSnafu {
1473                topic: "test_topic",
1474                partition: 0,
1475                offset: 1024u64,
1476            })
1477            .unwrap_err();
1478        let build_partition_client_err = Err::<(), _>(retry_failed_kafka_error())
1479            .context(BuildPartitionClientSnafu {
1480                topic: "test_topic",
1481                partition: 0,
1482            })
1483            .unwrap_err();
1484        let get_offset_err = Err::<(), _>(retry_failed_kafka_error())
1485            .context(GetOffsetSnafu {
1486                topic: "test_topic",
1487            })
1488            .unwrap_err();
1489
1490        assert!(delete_records_err.is_retryable());
1491        assert!(build_partition_client_err.is_retryable());
1492        assert!(get_offset_err.is_retryable());
1493        assert!(delete_records_err.retry_hint().is_retryable());
1494        assert!(build_partition_client_err.retry_hint().is_retryable());
1495        assert!(get_offset_err.retry_hint().is_retryable());
1496    }
1497
1498    #[test]
1499    fn test_kafka_non_retry_failed_errors_are_not_retryable() {
1500        let delete_records_err = Err::<(), _>(KafkaClientError::InvalidResponse("invalid".into()))
1501            .context(DeleteRecordsSnafu {
1502                topic: "test_topic",
1503                partition: 0,
1504                offset: 1024u64,
1505            })
1506            .unwrap_err();
1507        let build_partition_client_err =
1508            Err::<(), _>(KafkaClientError::InvalidResponse("invalid".into()))
1509                .context(BuildPartitionClientSnafu {
1510                    topic: "test_topic",
1511                    partition: 0,
1512                })
1513                .unwrap_err();
1514        let get_offset_err = Err::<(), _>(KafkaClientError::InvalidResponse("invalid".into()))
1515            .context(GetOffsetSnafu {
1516                topic: "test_topic",
1517            })
1518            .unwrap_err();
1519
1520        assert!(!delete_records_err.is_retryable());
1521        assert!(!build_partition_client_err.is_retryable());
1522        assert!(!get_offset_err.is_retryable());
1523        assert!(!delete_records_err.retry_hint().is_retryable());
1524        assert!(!build_partition_client_err.retry_hint().is_retryable());
1525        assert!(!get_offset_err.retry_hint().is_retryable());
1526    }
1527}