common_meta/
error.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::str::Utf8Error;
16use std::sync::Arc;
17
18use common_error::ext::{BoxedError, ErrorExt, RetryHint};
19use common_error::status_code::StatusCode;
20use common_macro::stack_trace_debug;
21use common_procedure::ProcedureId;
22use common_wal::kafka::rskafka_client_error_to_retry_hint;
23use object_store::error::retry_hint_from_opendal_error;
24use serde_json::error::Error as JsonError;
25use snafu::{Location, Snafu};
26use store_api::storage::RegionId;
27use table::metadata::TableId;
28
29use crate::DatanodeId;
30use crate::peer::Peer;
31
32mod retry_hint;
33
34pub use retry_hint::retry_hint_from_etcd_error;
35#[cfg(feature = "mysql_kvbackend")]
36pub use retry_hint::retry_hint_from_sqlx_error;
37#[cfg(feature = "pg_kvbackend")]
38pub use retry_hint::{retry_hint_from_postgres_error, retry_hint_from_postgres_pool_error};
39
40#[derive(Snafu)]
41#[snafu(visibility(pub))]
42#[stack_trace_debug]
43pub enum Error {
44    #[snafu(display("Empty key is not allowed"))]
45    EmptyKey {
46        #[snafu(implicit)]
47        location: Location,
48    },
49
50    #[snafu(display(
51        "Another procedure is operating the region: {} on peer: {}",
52        region_id,
53        peer_id
54    ))]
55    RegionOperatingRace {
56        #[snafu(implicit)]
57        location: Location,
58        peer_id: DatanodeId,
59        region_id: RegionId,
60    },
61
62    #[snafu(display("Failed to connect to Etcd"))]
63    ConnectEtcd {
64        #[snafu(source)]
65        error: etcd_client::Error,
66        #[snafu(implicit)]
67        location: Location,
68    },
69
70    #[snafu(display("Failed to execute via Etcd"))]
71    EtcdFailed {
72        #[snafu(source)]
73        error: etcd_client::Error,
74        #[snafu(implicit)]
75        location: Location,
76    },
77
78    #[snafu(display("Failed to execute {} txn operations via Etcd", max_operations))]
79    EtcdTxnFailed {
80        max_operations: usize,
81        #[snafu(source)]
82        error: etcd_client::Error,
83        #[snafu(implicit)]
84        location: Location,
85    },
86
87    #[snafu(display("Failed to get sequence: {}", err_msg))]
88    NextSequence {
89        err_msg: String,
90        #[snafu(implicit)]
91        location: Location,
92    },
93
94    #[snafu(display("Unexpected sequence value: {}", err_msg))]
95    UnexpectedSequenceValue {
96        err_msg: String,
97        #[snafu(implicit)]
98        location: Location,
99    },
100
101    #[snafu(display("Table info not found: {}", table))]
102    TableInfoNotFound {
103        table: String,
104        #[snafu(implicit)]
105        location: Location,
106    },
107
108    #[snafu(display("Failed to register procedure loader, type name: {}", type_name))]
109    RegisterProcedureLoader {
110        type_name: String,
111        #[snafu(implicit)]
112        location: Location,
113        source: common_procedure::error::Error,
114    },
115
116    #[snafu(display("Failed to register repartition procedure loader"))]
117    RegisterRepartitionProcedureLoader {
118        #[snafu(implicit)]
119        location: Location,
120        source: BoxedError,
121    },
122
123    #[snafu(display("Failed to create repartition procedure"))]
124    CreateRepartitionProcedure {
125        source: BoxedError,
126        #[snafu(implicit)]
127        location: Location,
128    },
129
130    #[snafu(display("Failed to persist the repartition GC requirement"))]
131    PersistRepartitionGcRequirement {
132        source: BoxedError,
133        #[snafu(implicit)]
134        location: Location,
135    },
136
137    #[snafu(display("Failed to submit procedure"))]
138    SubmitProcedure {
139        #[snafu(implicit)]
140        location: Location,
141        source: common_procedure::Error,
142    },
143
144    #[snafu(display("Failed to query procedure"))]
145    QueryProcedure {
146        #[snafu(implicit)]
147        location: Location,
148        source: common_procedure::Error,
149    },
150
151    #[snafu(display("Procedure not found: {pid}"))]
152    ProcedureNotFound {
153        #[snafu(implicit)]
154        location: Location,
155        pid: String,
156    },
157
158    #[snafu(display("Failed to parse procedure id: {key}"))]
159    ParseProcedureId {
160        #[snafu(implicit)]
161        location: Location,
162        key: String,
163        #[snafu(source)]
164        error: common_procedure::ParseIdError,
165    },
166
167    #[snafu(display("Unsupported operation {}", operation))]
168    Unsupported {
169        operation: String,
170        #[snafu(implicit)]
171        location: Location,
172    },
173
174    #[snafu(display("Trying to write to a read-only kv backend: {}", name))]
175    ReadOnlyKvBackend {
176        name: String,
177        #[snafu(implicit)]
178        location: Location,
179    },
180
181    #[snafu(display("Failed to get procedure state receiver, procedure id: {procedure_id}"))]
182    ProcedureStateReceiver {
183        procedure_id: ProcedureId,
184        #[snafu(implicit)]
185        location: Location,
186        source: common_procedure::Error,
187    },
188
189    #[snafu(display("Procedure state receiver not found: {procedure_id}"))]
190    ProcedureStateReceiverNotFound {
191        procedure_id: ProcedureId,
192        #[snafu(implicit)]
193        location: Location,
194    },
195
196    #[snafu(display("Failed to wait procedure done"))]
197    WaitProcedure {
198        #[snafu(implicit)]
199        location: Location,
200        source: common_procedure::Error,
201    },
202
203    #[snafu(display("Failed to start procedure manager"))]
204    StartProcedureManager {
205        #[snafu(implicit)]
206        location: Location,
207        source: common_procedure::Error,
208    },
209
210    #[snafu(display("Failed to stop procedure manager"))]
211    StopProcedureManager {
212        #[snafu(implicit)]
213        location: Location,
214        source: common_procedure::Error,
215    },
216
217    #[snafu(display(
218        "Failed to get procedure output, procedure id: {procedure_id}, error: {err_msg}"
219    ))]
220    ProcedureOutput {
221        procedure_id: String,
222        err_msg: String,
223        #[snafu(implicit)]
224        location: Location,
225    },
226
227    #[snafu(display("Primary key '{key}' not found when creating region request"))]
228    PrimaryKeyNotFound {
229        key: String,
230        #[snafu(implicit)]
231        location: Location,
232    },
233
234    #[snafu(display("Failed to build table meta for table: {}", table_name))]
235    BuildTableMeta {
236        table_name: String,
237        #[snafu(source)]
238        error: table::metadata::TableMetaBuilderError,
239        #[snafu(implicit)]
240        location: Location,
241    },
242
243    #[snafu(display("Table occurs error"))]
244    Table {
245        #[snafu(implicit)]
246        location: Location,
247        source: table::error::Error,
248    },
249
250    #[snafu(display("Failed to find table route for table id {}", table_id))]
251    TableRouteNotFound {
252        table_id: TableId,
253        #[snafu(implicit)]
254        location: Location,
255    },
256
257    #[snafu(display("Failed to find table repartition metadata for table id {}", table_id))]
258    TableRepartNotFound {
259        table_id: TableId,
260        #[snafu(implicit)]
261        location: Location,
262    },
263
264    #[snafu(display("Failed to decode protobuf"))]
265    DecodeProto {
266        #[snafu(implicit)]
267        location: Location,
268        #[snafu(source)]
269        error: prost::DecodeError,
270    },
271
272    #[snafu(display("Failed to encode object into json"))]
273    EncodeJson {
274        #[snafu(implicit)]
275        location: Location,
276        #[snafu(source)]
277        error: JsonError,
278    },
279
280    #[snafu(display("Failed to decode object from json"))]
281    DecodeJson {
282        #[snafu(implicit)]
283        location: Location,
284        #[snafu(source)]
285        error: JsonError,
286    },
287
288    #[snafu(display("Failed to serialize to json: {}", input))]
289    SerializeToJson {
290        input: String,
291        #[snafu(source)]
292        error: serde_json::error::Error,
293        #[snafu(implicit)]
294        location: Location,
295    },
296
297    #[snafu(display("Failed to deserialize from json: {}", input))]
298    DeserializeFromJson {
299        input: String,
300        #[snafu(source)]
301        error: serde_json::error::Error,
302        #[snafu(implicit)]
303        location: Location,
304    },
305
306    #[snafu(display("Payload not exist"))]
307    PayloadNotExist {
308        #[snafu(implicit)]
309        location: Location,
310    },
311
312    #[snafu(display("Failed to serde json"))]
313    SerdeJson {
314        #[snafu(source)]
315        error: serde_json::error::Error,
316        #[snafu(implicit)]
317        location: Location,
318    },
319
320    #[snafu(display("Failed to parse value {} into key {}", value, key))]
321    ParseOption {
322        key: String,
323        value: String,
324        #[snafu(implicit)]
325        location: Location,
326    },
327
328    #[snafu(display(
329        "Conflicting schema options: {}={} and {}={}",
330        first_key,
331        first_value,
332        second_key,
333        second_value
334    ))]
335    ConflictingSchemaOptions {
336        first_key: String,
337        first_value: String,
338        second_key: String,
339        second_value: String,
340        #[snafu(implicit)]
341        location: Location,
342    },
343
344    #[snafu(display("Corrupted table route data, err: {}", err_msg))]
345    RouteInfoCorrupted {
346        err_msg: String,
347        #[snafu(implicit)]
348        location: Location,
349    },
350
351    #[snafu(display("Illegal state from server, code: {}, error: {}", code, err_msg))]
352    IllegalServerState {
353        code: i32,
354        err_msg: String,
355        #[snafu(implicit)]
356        location: Location,
357    },
358
359    #[snafu(display("Failed to convert alter table request"))]
360    ConvertAlterTableRequest {
361        source: common_grpc_expr::error::Error,
362        #[snafu(implicit)]
363        location: Location,
364    },
365
366    #[snafu(display("Invalid protobuf message: {err_msg}"))]
367    InvalidProtoMsg {
368        err_msg: String,
369        #[snafu(implicit)]
370        location: Location,
371    },
372
373    #[snafu(display("Unexpected: {err_msg}"))]
374    Unexpected {
375        err_msg: String,
376        #[snafu(implicit)]
377        location: Location,
378    },
379
380    #[snafu(display("Metasrv election has no leader at this moment"))]
381    ElectionNoLeader {
382        #[snafu(implicit)]
383        location: Location,
384    },
385
386    #[snafu(display("Metasrv election leader lease expired"))]
387    ElectionLeaderLeaseExpired {
388        #[snafu(implicit)]
389        location: Location,
390    },
391
392    #[snafu(display("Metasrv election leader lease changed during election"))]
393    ElectionLeaderLeaseChanged {
394        #[snafu(implicit)]
395        location: Location,
396    },
397
398    #[snafu(display("Table already exists, table: {}", table_name))]
399    TableAlreadyExists {
400        table_name: String,
401        #[snafu(implicit)]
402        location: Location,
403    },
404
405    #[snafu(display(
406        "Cannot drop table '{}': an older tombstone already uses the same full name",
407        table_name
408    ))]
409    /// Raised when a live table is recreated with a name still reserved by an older tombstone.
410    TableNameTombstoneConflict {
411        table_name: String,
412        existing_table_id: TableId,
413        dropping_table_id: TableId,
414        #[snafu(implicit)]
415        location: Location,
416    },
417
418    #[snafu(display("View already exists, view: {}", view_name))]
419    ViewAlreadyExists {
420        view_name: String,
421        #[snafu(implicit)]
422        location: Location,
423    },
424
425    #[snafu(display("Flow already exists: {}", flow_name))]
426    FlowAlreadyExists {
427        flow_name: String,
428        #[snafu(implicit)]
429        location: Location,
430    },
431
432    #[snafu(display("Schema already exists, catalog:{}, schema: {}", catalog, schema))]
433    SchemaAlreadyExists {
434        catalog: String,
435        schema: String,
436        #[snafu(implicit)]
437        location: Location,
438    },
439
440    #[snafu(display("Failed to convert raw key to str"))]
441    ConvertRawKey {
442        #[snafu(implicit)]
443        location: Location,
444        #[snafu(source)]
445        error: Utf8Error,
446    },
447
448    #[snafu(display("Table not found: '{}'", table_name))]
449    TableNotFound {
450        table_name: String,
451        #[snafu(implicit)]
452        location: Location,
453    },
454
455    #[snafu(display("Region not found: {}", region_id))]
456    RegionNotFound {
457        region_id: RegionId,
458        #[snafu(implicit)]
459        location: Location,
460    },
461
462    #[snafu(display("View not found: '{}'", view_name))]
463    ViewNotFound {
464        view_name: String,
465        #[snafu(implicit)]
466        location: Location,
467    },
468
469    #[snafu(display("Flow not found: '{}'", flow_name))]
470    FlowNotFound {
471        flow_name: String,
472        #[snafu(implicit)]
473        location: Location,
474    },
475
476    #[snafu(display("Flow route not found: '{}'", flow_name))]
477    FlowRouteNotFound {
478        flow_name: String,
479        #[snafu(implicit)]
480        location: Location,
481    },
482
483    #[snafu(display("Schema nod found, schema: {}", table_schema))]
484    SchemaNotFound {
485        table_schema: String,
486        #[snafu(implicit)]
487        location: Location,
488    },
489
490    #[snafu(display("Catalog not found, catalog: {}", catalog))]
491    CatalogNotFound {
492        catalog: String,
493        #[snafu(implicit)]
494        location: Location,
495    },
496
497    #[snafu(display("Invalid metadata, err: {}", err_msg))]
498    InvalidMetadata {
499        err_msg: String,
500        #[snafu(implicit)]
501        location: Location,
502    },
503
504    #[snafu(display("Invalid view info, err: {}", err_msg))]
505    InvalidViewInfo {
506        err_msg: String,
507        #[snafu(implicit)]
508        location: Location,
509    },
510
511    #[snafu(display("Invalid flow request body: {:?}", body))]
512    InvalidFlowRequestBody {
513        body: Box<Option<api::v1::flow::flow_request::Body>>,
514        #[snafu(implicit)]
515        location: Location,
516    },
517
518    #[snafu(display("Failed to get kv cache, err: {}", err_msg))]
519    GetKvCache { err_msg: String },
520
521    #[snafu(display("Get null from cache, key: {}", key))]
522    CacheNotGet {
523        key: String,
524        #[snafu(implicit)]
525        location: Location,
526    },
527
528    #[snafu(display("Etcd txn error: {err_msg}"))]
529    EtcdTxnOpResponse {
530        err_msg: String,
531        #[snafu(implicit)]
532        location: Location,
533    },
534
535    #[snafu(display("External error"))]
536    External {
537        #[snafu(implicit)]
538        location: Location,
539        source: BoxedError,
540    },
541
542    #[snafu(display("The response exceeded size limit"))]
543    ResponseExceededSizeLimit {
544        #[snafu(implicit)]
545        location: Location,
546        source: BoxedError,
547    },
548
549    #[snafu(display("Invalid heartbeat response"))]
550    InvalidHeartbeatResponse {
551        #[snafu(implicit)]
552        location: Location,
553    },
554
555    #[snafu(display("Failed to operate on datanode: {}", peer))]
556    OperateDatanode {
557        #[snafu(implicit)]
558        location: Location,
559        peer: Peer,
560        source: BoxedError,
561    },
562
563    #[snafu(display("Retry later"))]
564    RetryLater {
565        source: BoxedError,
566        clean_poisons: bool,
567    },
568
569    #[snafu(display("Abort procedure"))]
570    AbortProcedure {
571        #[snafu(implicit)]
572        location: Location,
573        source: BoxedError,
574        clean_poisons: bool,
575    },
576
577    #[snafu(display("Failed to serialize WAL options for region: {region_id}"))]
578    SerializeWalOptions {
579        region_id: RegionId,
580        #[snafu(source)]
581        error: serde_json::Error,
582        #[snafu(implicit)]
583        location: Location,
584    },
585
586    #[snafu(display("Invalid number of topics {}", num_topics))]
587    InvalidNumTopics {
588        num_topics: usize,
589        #[snafu(implicit)]
590        location: Location,
591    },
592
593    #[snafu(display(
594        "Failed to build a Kafka client, broker endpoints: {:?}",
595        broker_endpoints
596    ))]
597    BuildKafkaClient {
598        broker_endpoints: Vec<String>,
599        #[snafu(implicit)]
600        location: Location,
601        #[snafu(source)]
602        error: rskafka::client::error::Error,
603    },
604
605    #[snafu(display("Failed to create TLS Config"))]
606    TlsConfig {
607        #[snafu(implicit)]
608        location: Location,
609        source: common_wal::error::Error,
610    },
611
612    #[snafu(display("Failed to build a Kafka controller client"))]
613    BuildKafkaCtrlClient {
614        #[snafu(implicit)]
615        location: Location,
616        #[snafu(source)]
617        error: rskafka::client::error::Error,
618    },
619
620    #[snafu(display(
621        "Failed to get a Kafka partition client, topic: {}, partition: {}",
622        topic,
623        partition
624    ))]
625    KafkaPartitionClient {
626        topic: String,
627        partition: i32,
628        #[snafu(implicit)]
629        location: Location,
630        #[snafu(source)]
631        error: rskafka::client::error::Error,
632    },
633
634    #[snafu(display(
635        "Failed to get offset from Kafka, topic: {}, partition: {}",
636        topic,
637        partition
638    ))]
639    KafkaGetOffset {
640        topic: String,
641        partition: i32,
642        #[snafu(implicit)]
643        location: Location,
644        #[snafu(source)]
645        error: rskafka::client::error::Error,
646    },
647
648    #[snafu(display("Failed to produce records to Kafka, topic: {}", topic))]
649    ProduceRecord {
650        topic: String,
651        #[snafu(implicit)]
652        location: Location,
653        #[snafu(source)]
654        error: rskafka::client::error::Error,
655    },
656
657    #[snafu(display("Failed to create a Kafka wal topic"))]
658    CreateKafkaWalTopic {
659        #[snafu(implicit)]
660        location: Location,
661        #[snafu(source)]
662        error: rskafka::client::error::Error,
663    },
664
665    #[snafu(display("The topic pool is empty"))]
666    EmptyTopicPool {
667        #[snafu(implicit)]
668        location: Location,
669    },
670
671    #[snafu(display("Unexpected table route type: {}", err_msg))]
672    UnexpectedLogicalRouteTable {
673        #[snafu(implicit)]
674        location: Location,
675        err_msg: String,
676    },
677
678    #[snafu(display("The tasks of {} cannot be empty", name))]
679    EmptyDdlTasks {
680        name: String,
681        #[snafu(implicit)]
682        location: Location,
683    },
684
685    #[snafu(display("Metadata corruption: {}", err_msg))]
686    MetadataCorruption {
687        err_msg: String,
688        #[snafu(implicit)]
689        location: Location,
690    },
691
692    #[snafu(display("Alter logical tables invalid arguments: {}", err_msg))]
693    AlterLogicalTablesInvalidArguments {
694        err_msg: String,
695        #[snafu(implicit)]
696        location: Location,
697    },
698
699    #[snafu(display("Create logical tables invalid arguments: {}", err_msg))]
700    CreateLogicalTablesInvalidArguments {
701        err_msg: String,
702        #[snafu(implicit)]
703        location: Location,
704    },
705
706    #[snafu(display("Invalid node info key: {}", key))]
707    InvalidNodeInfoKey {
708        key: String,
709        #[snafu(implicit)]
710        location: Location,
711    },
712
713    #[snafu(display("Invalid node stat key: {}", key))]
714    InvalidStatKey {
715        key: String,
716        #[snafu(implicit)]
717        location: Location,
718    },
719
720    #[snafu(display("Failed to parse number: {}", err_msg))]
721    ParseNum {
722        err_msg: String,
723        #[snafu(source)]
724        error: std::num::ParseIntError,
725        #[snafu(implicit)]
726        location: Location,
727    },
728
729    #[snafu(display("Invalid role: {}", role))]
730    InvalidRole {
731        role: i32,
732        #[snafu(implicit)]
733        location: Location,
734    },
735
736    #[snafu(display("Invalid set database option, key: {}, value: {}", key, value))]
737    InvalidSetDatabaseOption {
738        key: String,
739        value: String,
740        #[snafu(implicit)]
741        location: Location,
742    },
743
744    #[snafu(display("Invalid unset database option, key: {}", key))]
745    InvalidUnsetDatabaseOption {
746        key: String,
747        #[snafu(implicit)]
748        location: Location,
749    },
750
751    #[snafu(display("Invalid prefix: {}, key: {}", prefix, key))]
752    MismatchPrefix {
753        prefix: String,
754        key: String,
755        #[snafu(implicit)]
756        location: Location,
757    },
758
759    #[snafu(display("Failed to move values: {err_msg}"))]
760    MoveValues {
761        err_msg: String,
762        #[snafu(implicit)]
763        location: Location,
764    },
765
766    #[snafu(display("Failed to restore tombstone, target key already exists: {key}"))]
767    TombstoneTargetAlreadyExists {
768        key: String,
769        #[snafu(implicit)]
770        location: Location,
771    },
772
773    #[snafu(display("Failed to parse {} from utf8", name))]
774    FromUtf8 {
775        name: String,
776        #[snafu(source)]
777        error: std::string::FromUtf8Error,
778        #[snafu(implicit)]
779        location: Location,
780    },
781
782    #[snafu(display("Value not exists"))]
783    ValueNotExist {
784        #[snafu(implicit)]
785        location: Location,
786    },
787
788    #[snafu(display("Failed to get cache"))]
789    GetCache { source: Arc<Error> },
790
791    #[snafu(display(
792        "Failed to get latest cache value after {} attempts due to concurrent invalidation",
793        attempts
794    ))]
795    GetLatestCacheRetryExceeded {
796        attempts: usize,
797        #[snafu(implicit)]
798        location: Location,
799    },
800
801    #[cfg(feature = "pg_kvbackend")]
802    #[snafu(display("Failed to execute via Postgres, sql: {}", sql))]
803    PostgresExecution {
804        sql: String,
805        #[snafu(source)]
806        error: tokio_postgres::Error,
807        #[snafu(implicit)]
808        location: Location,
809    },
810
811    #[cfg(feature = "pg_kvbackend")]
812    #[snafu(display("Failed to create connection pool for Postgres"))]
813    CreatePostgresPool {
814        #[snafu(source)]
815        error: deadpool_postgres::CreatePoolError,
816        #[snafu(implicit)]
817        location: Location,
818    },
819
820    #[cfg(feature = "pg_kvbackend")]
821    #[snafu(display("Failed to get Postgres connection from pool: {}", reason))]
822    GetPostgresConnection {
823        reason: String,
824        #[snafu(implicit)]
825        location: Location,
826    },
827
828    #[cfg(feature = "pg_kvbackend")]
829    #[snafu(display("Failed to get Postgres client"))]
830    GetPostgresClient {
831        #[snafu(source)]
832        error: deadpool::managed::PoolError<tokio_postgres::Error>,
833        #[snafu(implicit)]
834        location: Location,
835    },
836
837    #[cfg(feature = "pg_kvbackend")]
838    #[snafu(display("Failed to {} Postgres transaction", operation))]
839    PostgresTransaction {
840        #[snafu(source)]
841        error: tokio_postgres::Error,
842        #[snafu(implicit)]
843        location: Location,
844        operation: String,
845    },
846
847    #[cfg(feature = "pg_kvbackend")]
848    #[snafu(display("Failed to setup PostgreSQL TLS configuration: {}", reason))]
849    PostgresTlsConfig {
850        reason: String,
851        #[snafu(implicit)]
852        location: Location,
853    },
854
855    #[snafu(display("Failed to load TLS certificate from path: {}", path))]
856    LoadTlsCertificate {
857        path: String,
858        #[snafu(source)]
859        error: std::io::Error,
860        #[snafu(implicit)]
861        location: Location,
862    },
863
864    #[cfg(feature = "pg_kvbackend")]
865    #[snafu(display("Invalid TLS configuration: {}", reason))]
866    InvalidTlsConfig {
867        reason: String,
868        #[snafu(implicit)]
869        location: Location,
870    },
871
872    #[cfg(feature = "mysql_kvbackend")]
873    #[snafu(display("Failed to execute via MySql, sql: {}", sql))]
874    MySqlExecution {
875        sql: String,
876        #[snafu(source)]
877        error: sqlx::Error,
878        #[snafu(implicit)]
879        location: Location,
880    },
881
882    #[cfg(feature = "mysql_kvbackend")]
883    #[snafu(display("Failed to create connection pool for MySql"))]
884    CreateMySqlPool {
885        #[snafu(source)]
886        error: sqlx::Error,
887        #[snafu(implicit)]
888        location: Location,
889    },
890
891    #[cfg(feature = "mysql_kvbackend")]
892    #[snafu(display("Failed to decode sql value"))]
893    DecodeSqlValue {
894        #[snafu(source)]
895        error: sqlx::error::Error,
896        #[snafu(implicit)]
897        location: Location,
898    },
899
900    #[cfg(feature = "mysql_kvbackend")]
901    #[snafu(display("Failed to acquire mysql client from pool"))]
902    AcquireMySqlClient {
903        #[snafu(source)]
904        error: sqlx::Error,
905        #[snafu(implicit)]
906        location: Location,
907    },
908
909    #[cfg(feature = "mysql_kvbackend")]
910    #[snafu(display("Failed to {} MySql transaction", operation))]
911    MySqlTransaction {
912        #[snafu(source)]
913        error: sqlx::Error,
914        #[snafu(implicit)]
915        location: Location,
916        operation: String,
917    },
918
919    #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
920    #[snafu(display("Rds transaction retry failed"))]
921    RdsTransactionRetryFailed {
922        #[snafu(implicit)]
923        location: Location,
924    },
925
926    #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
927    #[snafu(display("Sql execution timeout, sql: {}, duration: {:?}", sql, duration))]
928    SqlExecutionTimeout {
929        sql: String,
930        duration: std::time::Duration,
931        #[snafu(implicit)]
932        location: Location,
933    },
934
935    #[snafu(display(
936        "Datanode table info not found, table id: {}, datanode id: {}",
937        table_id,
938        datanode_id
939    ))]
940    DatanodeTableInfoNotFound {
941        datanode_id: DatanodeId,
942        table_id: TableId,
943        #[snafu(implicit)]
944        location: Location,
945    },
946
947    #[snafu(display("Invalid topic name prefix: {}", prefix))]
948    InvalidTopicNamePrefix {
949        prefix: String,
950        #[snafu(implicit)]
951        location: Location,
952    },
953
954    #[snafu(display("No leader found for table_id: {}", table_id))]
955    NoLeader {
956        table_id: TableId,
957        #[snafu(implicit)]
958        location: Location,
959    },
960
961    #[snafu(display(
962        "Procedure poison key already exists with a different value, key: {}, value: {}",
963        key,
964        value
965    ))]
966    ProcedurePoisonConflict {
967        key: String,
968        value: String,
969        #[snafu(implicit)]
970        location: Location,
971    },
972
973    #[snafu(display("Failed to put poison, table metadata may be corrupted"))]
974    PutPoison {
975        #[snafu(implicit)]
976        location: Location,
977        #[snafu(source)]
978        source: common_procedure::error::Error,
979    },
980
981    #[snafu(display("Invalid file path: {}", file_path))]
982    InvalidFilePath {
983        #[snafu(implicit)]
984        location: Location,
985        file_path: String,
986    },
987
988    #[snafu(display("Failed to serialize flexbuffers"))]
989    SerializeFlexbuffers {
990        #[snafu(implicit)]
991        location: Location,
992        #[snafu(source)]
993        error: flexbuffers::SerializationError,
994    },
995
996    #[snafu(display("Failed to deserialize flexbuffers"))]
997    DeserializeFlexbuffers {
998        #[snafu(implicit)]
999        location: Location,
1000        #[snafu(source)]
1001        error: flexbuffers::DeserializationError,
1002    },
1003
1004    #[snafu(display("Failed to read flexbuffers"))]
1005    ReadFlexbuffers {
1006        #[snafu(implicit)]
1007        location: Location,
1008        #[snafu(source)]
1009        error: flexbuffers::ReaderError,
1010    },
1011
1012    #[snafu(display("Invalid file name: {}", reason))]
1013    InvalidFileName {
1014        #[snafu(implicit)]
1015        location: Location,
1016        reason: String,
1017    },
1018
1019    #[snafu(display("Invalid file extension: {}", reason))]
1020    InvalidFileExtension {
1021        #[snafu(implicit)]
1022        location: Location,
1023        reason: String,
1024    },
1025
1026    #[snafu(display("Failed to write object, file path: {}", file_path))]
1027    WriteObject {
1028        #[snafu(implicit)]
1029        location: Location,
1030        file_path: String,
1031        #[snafu(source)]
1032        error: object_store::Error,
1033    },
1034
1035    #[snafu(display("Failed to read object, file path: {}", file_path))]
1036    ReadObject {
1037        #[snafu(implicit)]
1038        location: Location,
1039        file_path: String,
1040        #[snafu(source)]
1041        error: object_store::Error,
1042    },
1043
1044    #[snafu(display("Missing column ids"))]
1045    MissingColumnIds {
1046        #[snafu(implicit)]
1047        location: Location,
1048    },
1049
1050    #[snafu(display(
1051        "Missing column in column metadata: {}, table: {}, table_id: {}",
1052        column_name,
1053        table_name,
1054        table_id,
1055    ))]
1056    MissingColumnInColumnMetadata {
1057        column_name: String,
1058        #[snafu(implicit)]
1059        location: Location,
1060        table_name: String,
1061        table_id: TableId,
1062    },
1063
1064    #[snafu(display(
1065        "Mismatch column id: column_name: {}, column_id: {}, table: {}, table_id: {}",
1066        column_name,
1067        column_id,
1068        table_name,
1069        table_id,
1070    ))]
1071    MismatchColumnId {
1072        column_name: String,
1073        column_id: u32,
1074        #[snafu(implicit)]
1075        location: Location,
1076        table_name: String,
1077        table_id: TableId,
1078    },
1079
1080    #[snafu(display("Failed to convert column def, column: {}", column))]
1081    ConvertColumnDef {
1082        column: String,
1083        #[snafu(implicit)]
1084        location: Location,
1085        source: api::error::Error,
1086    },
1087
1088    #[snafu(display("Failed to convert time ranges"))]
1089    ConvertTimeRanges {
1090        #[snafu(implicit)]
1091        location: Location,
1092        source: api::error::Error,
1093    },
1094
1095    #[snafu(display(
1096        "Column metadata inconsistencies found in table: {}, table_id: {}",
1097        table_name,
1098        table_id
1099    ))]
1100    ColumnMetadataConflicts {
1101        table_name: String,
1102        table_id: TableId,
1103    },
1104
1105    #[snafu(display(
1106        "Column not found in column metadata, column_name: {}, column_id: {}",
1107        column_name,
1108        column_id
1109    ))]
1110    ColumnNotFound { column_name: String, column_id: u32 },
1111
1112    #[snafu(display(
1113        "Column id mismatch, column_name: {}, expected column_id: {}, actual column_id: {}",
1114        column_name,
1115        expected_column_id,
1116        actual_column_id
1117    ))]
1118    ColumnIdMismatch {
1119        column_name: String,
1120        expected_column_id: u32,
1121        actual_column_id: u32,
1122    },
1123
1124    #[snafu(display(
1125        "Timestamp column mismatch, expected column_name: {}, expected column_id: {}, actual column_name: {}, actual column_id: {}",
1126        expected_column_name,
1127        expected_column_id,
1128        actual_column_name,
1129        actual_column_id,
1130    ))]
1131    TimestampMismatch {
1132        expected_column_name: String,
1133        expected_column_id: u32,
1134        actual_column_name: String,
1135        actual_column_id: u32,
1136    },
1137
1138    #[cfg(feature = "enterprise")]
1139    #[snafu(display("Too large duration"))]
1140    TooLargeDuration {
1141        #[snafu(source)]
1142        error: prost_types::DurationError,
1143        #[snafu(implicit)]
1144        location: Location,
1145    },
1146
1147    #[cfg(feature = "enterprise")]
1148    #[snafu(display("Negative duration"))]
1149    NegativeDuration {
1150        #[snafu(source)]
1151        error: prost_types::DurationError,
1152        #[snafu(implicit)]
1153        location: Location,
1154    },
1155
1156    #[cfg(feature = "enterprise")]
1157    #[snafu(display("Missing interval field"))]
1158    MissingInterval {
1159        #[snafu(implicit)]
1160        location: Location,
1161    },
1162}
1163
1164pub type Result<T> = std::result::Result<T, Error>;
1165
1166impl ErrorExt for Error {
1167    fn status_code(&self) -> StatusCode {
1168        use Error::*;
1169        match self {
1170            IllegalServerState { .. }
1171            | EtcdTxnOpResponse { .. }
1172            | EtcdFailed { .. }
1173            | EtcdTxnFailed { .. }
1174            | ConnectEtcd { .. }
1175            | MoveValues { .. }
1176            | TombstoneTargetAlreadyExists { .. }
1177            | GetCache { .. }
1178            | GetLatestCacheRetryExceeded { .. }
1179            | SerializeToJson { .. }
1180            | DeserializeFromJson { .. }
1181            | ElectionNoLeader { .. }
1182            | ElectionLeaderLeaseExpired { .. }
1183            | ElectionLeaderLeaseChanged { .. } => StatusCode::Internal,
1184
1185            NoLeader { .. } => StatusCode::TableUnavailable,
1186            ValueNotExist { .. }
1187            | ProcedurePoisonConflict { .. }
1188            | ProcedureStateReceiverNotFound { .. }
1189            | MissingColumnIds { .. }
1190            | MissingColumnInColumnMetadata { .. }
1191            | MismatchColumnId { .. }
1192            | ColumnMetadataConflicts { .. }
1193            | ColumnNotFound { .. }
1194            | ColumnIdMismatch { .. }
1195            | TimestampMismatch { .. } => StatusCode::Unexpected,
1196
1197            Unsupported { .. } | ReadOnlyKvBackend { .. } => StatusCode::Unsupported,
1198            WriteObject { .. } | ReadObject { .. } => StatusCode::StorageUnavailable,
1199
1200            SerdeJson { .. }
1201            | ParseOption { .. }
1202            | RouteInfoCorrupted { .. }
1203            | InvalidProtoMsg { .. }
1204            | InvalidMetadata { .. }
1205            | Unexpected { .. }
1206            | TableInfoNotFound { .. }
1207            | NextSequence { .. }
1208            | UnexpectedSequenceValue { .. }
1209            | InvalidHeartbeatResponse { .. }
1210            | EncodeJson { .. }
1211            | DecodeJson { .. }
1212            | PayloadNotExist { .. }
1213            | ConvertRawKey { .. }
1214            | DecodeProto { .. }
1215            | BuildTableMeta { .. }
1216            | TableRouteNotFound { .. }
1217            | TableRepartNotFound { .. }
1218            | RegionOperatingRace { .. }
1219            | SerializeWalOptions { .. }
1220            | BuildKafkaClient { .. }
1221            | BuildKafkaCtrlClient { .. }
1222            | KafkaPartitionClient { .. }
1223            | ProduceRecord { .. }
1224            | CreateKafkaWalTopic { .. }
1225            | EmptyTopicPool { .. }
1226            | UnexpectedLogicalRouteTable { .. }
1227            | ProcedureOutput { .. }
1228            | FromUtf8 { .. }
1229            | MetadataCorruption { .. }
1230            | KafkaGetOffset { .. }
1231            | ReadFlexbuffers { .. }
1232            | SerializeFlexbuffers { .. }
1233            | DeserializeFlexbuffers { .. }
1234            | ConvertTimeRanges { .. } => StatusCode::Unexpected,
1235
1236            GetKvCache { .. } | CacheNotGet { .. } => StatusCode::Internal,
1237
1238            SchemaAlreadyExists { .. } => StatusCode::DatabaseAlreadyExists,
1239
1240            ProcedureNotFound { .. }
1241            | InvalidViewInfo { .. }
1242            | PrimaryKeyNotFound { .. }
1243            | EmptyKey { .. }
1244            | AlterLogicalTablesInvalidArguments { .. }
1245            | CreateLogicalTablesInvalidArguments { .. }
1246            | MismatchPrefix { .. }
1247            | TlsConfig { .. }
1248            | InvalidSetDatabaseOption { .. }
1249            | InvalidUnsetDatabaseOption { .. }
1250            | InvalidTopicNamePrefix { .. }
1251            | InvalidFileExtension { .. }
1252            | InvalidFileName { .. }
1253            | InvalidFlowRequestBody { .. }
1254            | InvalidFilePath { .. }
1255            | ConflictingSchemaOptions { .. } => StatusCode::InvalidArguments,
1256
1257            #[cfg(feature = "enterprise")]
1258            MissingInterval { .. } | NegativeDuration { .. } | TooLargeDuration { .. } => {
1259                StatusCode::InvalidArguments
1260            }
1261
1262            FlowNotFound { .. } => StatusCode::FlowNotFound,
1263            FlowRouteNotFound { .. } => StatusCode::Unexpected,
1264            FlowAlreadyExists { .. } => StatusCode::FlowAlreadyExists,
1265
1266            ViewNotFound { .. } | TableNotFound { .. } | RegionNotFound { .. } => {
1267                StatusCode::TableNotFound
1268            }
1269            ViewAlreadyExists { .. }
1270            | TableAlreadyExists { .. }
1271            | TableNameTombstoneConflict { .. } => StatusCode::TableAlreadyExists,
1272
1273            SubmitProcedure { source, .. }
1274            | QueryProcedure { source, .. }
1275            | WaitProcedure { source, .. }
1276            | StartProcedureManager { source, .. }
1277            | StopProcedureManager { source, .. } => source.status_code(),
1278            RegisterProcedureLoader { source, .. } => source.status_code(),
1279            External { source, .. } => source.status_code(),
1280            ResponseExceededSizeLimit { source, .. } => source.status_code(),
1281            OperateDatanode { source, .. } => source.status_code(),
1282            Table { source, .. } => source.status_code(),
1283            RetryLater { source, .. } => source.status_code(),
1284            AbortProcedure { source, .. } => source.status_code(),
1285            ConvertAlterTableRequest { source, .. } => source.status_code(),
1286            PutPoison { source, .. } => source.status_code(),
1287            ConvertColumnDef { source, .. } => source.status_code(),
1288            ProcedureStateReceiver { source, .. } => source.status_code(),
1289            RegisterRepartitionProcedureLoader { source, .. } => source.status_code(),
1290            CreateRepartitionProcedure { source, .. } => source.status_code(),
1291            PersistRepartitionGcRequirement { source, .. } => source.status_code(),
1292
1293            ParseProcedureId { .. }
1294            | InvalidNumTopics { .. }
1295            | SchemaNotFound { .. }
1296            | CatalogNotFound { .. }
1297            | InvalidNodeInfoKey { .. }
1298            | InvalidStatKey { .. }
1299            | ParseNum { .. }
1300            | InvalidRole { .. }
1301            | EmptyDdlTasks { .. } => StatusCode::InvalidArguments,
1302
1303            LoadTlsCertificate { .. } => StatusCode::Internal,
1304
1305            #[cfg(feature = "pg_kvbackend")]
1306            PostgresExecution { .. }
1307            | CreatePostgresPool { .. }
1308            | GetPostgresConnection { .. }
1309            | GetPostgresClient { .. }
1310            | PostgresTransaction { .. }
1311            | PostgresTlsConfig { .. }
1312            | InvalidTlsConfig { .. } => StatusCode::Internal,
1313            #[cfg(feature = "mysql_kvbackend")]
1314            MySqlExecution { .. }
1315            | CreateMySqlPool { .. }
1316            | DecodeSqlValue { .. }
1317            | AcquireMySqlClient { .. }
1318            | MySqlTransaction { .. } => StatusCode::Internal,
1319            #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
1320            RdsTransactionRetryFailed { .. } | SqlExecutionTimeout { .. } => StatusCode::Internal,
1321            DatanodeTableInfoNotFound { .. } => StatusCode::Internal,
1322        }
1323    }
1324
1325    fn as_any(&self) -> &dyn std::any::Any {
1326        self
1327    }
1328
1329    fn retry_hint(&self) -> RetryHint {
1330        use Error::*;
1331
1332        match self {
1333            RetryLater { .. }
1334            | GetLatestCacheRetryExceeded { .. }
1335            | NoLeader { .. }
1336            | ElectionNoLeader { .. }
1337            | ElectionLeaderLeaseExpired { .. }
1338            | ElectionLeaderLeaseChanged { .. } => RetryHint::Retryable,
1339            ConnectEtcd { error, .. } | EtcdFailed { error, .. } | EtcdTxnFailed { error, .. } => {
1340                retry_hint_from_etcd_error(error)
1341            }
1342            WriteObject { error, .. } | ReadObject { error, .. } => {
1343                retry_hint_from_opendal_error(error)
1344            }
1345            BuildKafkaClient { error, .. }
1346            | BuildKafkaCtrlClient { error, .. }
1347            | KafkaPartitionClient { error, .. }
1348            | KafkaGetOffset { error, .. }
1349            | ProduceRecord { error, .. }
1350            | CreateKafkaWalTopic { error, .. } => rskafka_client_error_to_retry_hint(error),
1351            SubmitProcedure { source, .. }
1352            | QueryProcedure { source, .. }
1353            | WaitProcedure { source, .. }
1354            | StartProcedureManager { source, .. }
1355            | StopProcedureManager { source, .. }
1356            | RegisterProcedureLoader { source, .. }
1357            | PutPoison { source, .. }
1358            | ProcedureStateReceiver { source, .. } => source.retry_hint(),
1359            External { source, .. }
1360            | ResponseExceededSizeLimit { source, .. }
1361            | OperateDatanode { source, .. }
1362            | AbortProcedure { source, .. }
1363            | RegisterRepartitionProcedureLoader { source, .. }
1364            | CreateRepartitionProcedure { source, .. }
1365            | PersistRepartitionGcRequirement { source, .. } => source.retry_hint(),
1366            Table { source, .. } => source.retry_hint(),
1367            ConvertAlterTableRequest { source, .. } => source.retry_hint(),
1368            ConvertColumnDef { source, .. } => source.retry_hint(),
1369            GetCache { source, .. } => source.retry_hint(),
1370            #[cfg(feature = "pg_kvbackend")]
1371            PostgresExecution { error, .. } | PostgresTransaction { error, .. } => {
1372                retry_hint_from_postgres_error(error)
1373            }
1374            #[cfg(feature = "pg_kvbackend")]
1375            GetPostgresClient { error, .. } => retry_hint_from_postgres_pool_error(error),
1376            #[cfg(feature = "mysql_kvbackend")]
1377            MySqlExecution { error, .. }
1378            | CreateMySqlPool { error, .. }
1379            | AcquireMySqlClient { error, .. }
1380            | MySqlTransaction { error, .. } => retry_hint_from_sqlx_error(error),
1381            #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
1382            RdsTransactionRetryFailed { .. } | SqlExecutionTimeout { .. } => RetryHint::Retryable,
1383            _ => RetryHint::NonRetryable,
1384        }
1385    }
1386}
1387
1388impl Error {
1389    #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
1390    /// Check if the error is a serialization error.
1391    pub fn is_serialization_error(&self) -> bool {
1392        match self {
1393            #[cfg(feature = "pg_kvbackend")]
1394            Error::PostgresExecution { error, .. } | Error::PostgresTransaction { error, .. } => {
1395                retry_hint::is_postgres_serialization_error(error)
1396            }
1397            #[cfg(feature = "mysql_kvbackend")]
1398            Error::MySqlExecution { error, .. } | Error::MySqlTransaction { error, .. } => {
1399                retry_hint::is_mysql_serialization_error(error)
1400            }
1401            _ => false,
1402        }
1403    }
1404
1405    /// Creates a new [Error::RetryLater] error from source `err`.
1406    pub fn retry_later<E: ErrorExt + Send + Sync + 'static>(err: E) -> Error {
1407        Error::RetryLater {
1408            source: BoxedError::new(err),
1409            clean_poisons: false,
1410        }
1411    }
1412
1413    /// Determine whether it is a retry later type through [StatusCode]
1414    pub fn is_retry_later(&self) -> bool {
1415        matches!(
1416            self,
1417            Error::RetryLater { .. } | Error::GetLatestCacheRetryExceeded { .. }
1418        )
1419    }
1420
1421    /// Determine whether it needs to clean poisons.
1422    pub fn need_clean_poisons(&self) -> bool {
1423        matches!(
1424            self,
1425            Error::AbortProcedure { clean_poisons, .. } if *clean_poisons
1426        ) || matches!(
1427            self,
1428            Error::RetryLater { clean_poisons, .. } if *clean_poisons
1429        )
1430    }
1431
1432    /// Returns true if the response exceeds the size limit.
1433    pub fn is_exceeded_size_limit(&self) -> bool {
1434        match self {
1435            Error::EtcdFailed {
1436                error: etcd_client::Error::GRpcStatus(status),
1437                ..
1438            } => status.code() == tonic::Code::OutOfRange,
1439            Error::ResponseExceededSizeLimit { .. } => true,
1440            _ => false,
1441        }
1442    }
1443}
1444
1445#[cfg(test)]
1446mod retry_hint_tests {
1447    use std::sync::Arc;
1448
1449    use common_error::mock::MockError;
1450
1451    use super::*;
1452
1453    #[test]
1454    fn test_retry_later_hint_is_retryable() {
1455        let err = Error::retry_later(MockError::new(StatusCode::Internal));
1456
1457        assert_eq!(err.retry_hint(), RetryHint::Retryable);
1458    }
1459
1460    #[test]
1461    fn test_latest_cache_retry_exceeded_hint_is_retryable() {
1462        let err = GetLatestCacheRetryExceededSnafu { attempts: 3_usize }.build();
1463
1464        assert_eq!(err.retry_hint(), RetryHint::Retryable);
1465    }
1466
1467    #[test]
1468    fn test_get_cache_forwards_retry_hint() {
1469        let source = Arc::new(Error::retry_later(MockError::new(StatusCode::Internal)));
1470        let err = Error::GetCache { source };
1471
1472        assert_eq!(err.retry_hint(), RetryHint::Retryable);
1473    }
1474
1475    #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
1476    #[test]
1477    fn test_sql_execution_timeout_hint_is_retryable() {
1478        let err = SqlExecutionTimeoutSnafu {
1479            sql: "SELECT 1".to_string(),
1480            duration: std::time::Duration::from_secs(1),
1481        }
1482        .build();
1483
1484        assert_eq!(err.retry_hint(), RetryHint::Retryable);
1485    }
1486
1487    #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
1488    #[test]
1489    fn test_rds_transaction_retry_failed_hint_is_retryable() {
1490        let err = RdsTransactionRetryFailedSnafu.build();
1491
1492        assert_eq!(err.retry_hint(), RetryHint::Retryable);
1493    }
1494
1495    #[test]
1496    fn test_default_hint_is_non_retryable() {
1497        let err = UnexpectedSnafu {
1498            err_msg: "mock error",
1499        }
1500        .build();
1501
1502        assert_eq!(err.retry_hint(), RetryHint::NonRetryable);
1503    }
1504}
common_meta/error.rs

common_meta/
error.rs
