mito2/
error.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::any::Any;
16use std::sync::Arc;
17
18use common_datasource::compression::CompressionType;
19use common_error::ext::{BoxedError, ErrorExt, RetryHint};
20use common_error::status_code::StatusCode;
21use common_macro::stack_trace_debug;
22use common_memory_manager;
23use common_runtime::JoinError;
24use common_time::Timestamp;
25use common_time::timestamp::TimeUnit;
26use datatypes::arrow::error::ArrowError;
27use datatypes::prelude::ConcreteDataType;
28use object_store::ErrorKind;
29use object_store::error::retry_hint_from_opendal_error;
30use partition::error::Error as PartitionError;
31use prost::DecodeError;
32use snafu::{Location, Snafu};
33use store_api::ManifestVersion;
34use store_api::logstore::provider::Provider;
35use store_api::storage::{FileId, RegionId};
36use tokio::time::error::Elapsed;
37
38use crate::cache::file_cache::FileType;
39use crate::region::RegionRoleState;
40use crate::schedule::remote_job_scheduler::JobId;
41use crate::worker::WorkerId;
42
43#[derive(Snafu)]
44#[snafu(visibility(pub))]
45#[stack_trace_debug]
46pub enum Error {
47    #[snafu(display("Unexpected data type"))]
48    DataTypeMismatch {
49        source: datatypes::error::Error,
50        #[snafu(implicit)]
51        location: Location,
52    },
53
54    #[snafu(display("External error, context: {}", context))]
55    External {
56        source: BoxedError,
57        context: String,
58        #[snafu(implicit)]
59        location: Location,
60    },
61
62    #[snafu(display("OpenDAL operator failed"))]
63    OpenDal {
64        #[snafu(implicit)]
65        location: Location,
66        #[snafu(source)]
67        error: object_store::Error,
68    },
69
70    #[snafu(display(
71        "Manifest delta {} disappeared after it was listed, path: {}",
72        version,
73        path
74    ))]
75    ManifestDeltaNotFound {
76        version: ManifestVersion,
77        path: String,
78        #[snafu(source)]
79        error: object_store::Error,
80        #[snafu(implicit)]
81        location: Location,
82    },
83
84    #[snafu(display("Fail to compress object by {}, path: {}", compress_type, path))]
85    CompressObject {
86        compress_type: CompressionType,
87        path: String,
88        #[snafu(source)]
89        error: std::io::Error,
90    },
91
92    #[snafu(display("Fail to decompress object by {}, path: {}", compress_type, path))]
93    DecompressObject {
94        compress_type: CompressionType,
95        path: String,
96        #[snafu(source)]
97        error: std::io::Error,
98    },
99
100    #[snafu(display("Failed to ser/de json object"))]
101    SerdeJson {
102        #[snafu(implicit)]
103        location: Location,
104        #[snafu(source)]
105        error: serde_json::Error,
106    },
107
108    #[snafu(display("Failed to serialize column metadata"))]
109    SerializeColumnMetadata {
110        #[snafu(source)]
111        error: serde_json::Error,
112        #[snafu(implicit)]
113        location: Location,
114    },
115
116    #[snafu(display("Failed to serialize manifest, region_id: {}", region_id))]
117    SerializeManifest {
118        region_id: RegionId,
119        #[snafu(source)]
120        error: serde_json::Error,
121        #[snafu(implicit)]
122        location: Location,
123    },
124
125    #[snafu(display("Invalid scan index, start: {}, end: {}", start, end))]
126    InvalidScanIndex {
127        start: ManifestVersion,
128        end: ManifestVersion,
129        #[snafu(implicit)]
130        location: Location,
131    },
132
133    #[snafu(display("Invalid UTF-8 content"))]
134    Utf8 {
135        #[snafu(implicit)]
136        location: Location,
137        #[snafu(source)]
138        error: std::str::Utf8Error,
139    },
140
141    #[snafu(display("Cannot find RegionMetadata"))]
142    RegionMetadataNotFound {
143        #[snafu(implicit)]
144        location: Location,
145    },
146
147    #[snafu(display("Failed to join handle"))]
148    Join {
149        #[snafu(source)]
150        error: common_runtime::JoinError,
151        #[snafu(implicit)]
152        location: Location,
153    },
154
155    #[snafu(display("Worker {} is stopped", id))]
156    WorkerStopped {
157        id: WorkerId,
158        #[snafu(implicit)]
159        location: Location,
160    },
161
162    #[snafu(display("Failed to recv result"))]
163    Recv {
164        #[snafu(source)]
165        error: tokio::sync::oneshot::error::RecvError,
166        #[snafu(implicit)]
167        location: Location,
168    },
169
170    #[snafu(display("Invalid metadata, {}", reason))]
171    InvalidMeta {
172        reason: String,
173        #[snafu(implicit)]
174        location: Location,
175    },
176
177    #[snafu(display("Invalid region metadata"))]
178    InvalidMetadata {
179        source: store_api::metadata::MetadataError,
180        #[snafu(implicit)]
181        location: Location,
182    },
183
184    #[snafu(display("Failed to create RecordBatch from vectors"))]
185    NewRecordBatch {
186        #[snafu(implicit)]
187        location: Location,
188        #[snafu(source)]
189        error: ArrowError,
190    },
191
192    #[snafu(display("Failed to read parquet file, path: {}", path))]
193    ReadParquet {
194        path: String,
195        #[snafu(source)]
196        error: parquet::errors::ParquetError,
197        #[snafu(implicit)]
198        location: Location,
199    },
200
201    #[snafu(display("Failed to write parquet file"))]
202    WriteParquet {
203        #[snafu(source)]
204        error: parquet::errors::ParquetError,
205        #[snafu(implicit)]
206        location: Location,
207    },
208
209    #[snafu(display(
210        "Cannot assign a stable field id to native histogram sub-field '{}' of column id {} (unknown sub-field name or derived id overflows i32)",
211        field_name,
212        column_id
213    ))]
214    InvalidNativeHistogramSubfield {
215        column_id: i32,
216        field_name: String,
217        #[snafu(implicit)]
218        location: Location,
219    },
220
221    #[snafu(display(
222        "Native histogram column '{}' has no usable PARQUET:field_id to namespace its sub-field ids (missing, malformed, or exceeds i32::MAX)",
223        field_name
224    ))]
225    InvalidNativeHistogramFieldId {
226        field_name: String,
227        #[snafu(implicit)]
228        location: Location,
229    },
230
231    #[snafu(display("Region {} not found", region_id))]
232    RegionNotFound {
233        region_id: RegionId,
234        #[snafu(implicit)]
235        location: Location,
236    },
237
238    #[snafu(display("Object store not found: {}", object_store))]
239    ObjectStoreNotFound {
240        object_store: String,
241        #[snafu(implicit)]
242        location: Location,
243    },
244
245    #[snafu(display("Region {} is corrupted, reason: {}", region_id, reason))]
246    RegionCorrupted {
247        region_id: RegionId,
248        reason: String,
249        #[snafu(implicit)]
250        location: Location,
251    },
252
253    #[snafu(display("Invalid request to region {}, reason: {}", region_id, reason))]
254    InvalidRequest {
255        region_id: RegionId,
256        reason: String,
257        #[snafu(implicit)]
258        location: Location,
259    },
260
261    #[snafu(display(
262        "STALE_CURSOR: incremental query stale, region: {}, given_seq: {}, min_readable_seq: {}, retry_hint: FALLBACK_FULL_RECOMPUTE",
263        region_id,
264        given_seq,
265        min_readable_seq
266    ))]
267    IncrementalQueryStale {
268        region_id: RegionId,
269        given_seq: u64,
270        min_readable_seq: u64,
271        #[snafu(implicit)]
272        location: Location,
273    },
274
275    #[snafu(display(
276        "STALE_SNAPSHOT_FENCE: snapshot upper bound stale, region: {}, given_seq: {}, min_enforceable_seq: {}, retry_hint: REBIND_SNAPSHOT_FENCE",
277        region_id,
278        given_seq,
279        min_enforceable_seq
280    ))]
281    SnapshotFenceStale {
282        region_id: RegionId,
283        given_seq: u64,
284        min_enforceable_seq: u64,
285        #[snafu(implicit)]
286        location: Location,
287    },
288
289    #[snafu(display(
290        "SEQUENCE_RANGE_UNSUPPORTED: exact sequence-range read unsupported, region: {}, min_seq: {}, max_seq: {}, reason: {}, retry_hint: FALLBACK_MEMTABLE_ONLY_OR_FULL_RECOMPUTE",
291        region_id,
292        min_seq,
293        max_seq,
294        reason
295    ))]
296    SequenceRangeUnsupported {
297        region_id: RegionId,
298        min_seq: u64,
299        max_seq: u64,
300        reason: String,
301        #[snafu(implicit)]
302        location: Location,
303    },
304
305    #[snafu(display(
306        "region {} is unusable for sequence reads: file {} declares region {}",
307        region_id,
308        file_id,
309        file_region_id
310    ))]
311    RegionSequenceDomainBroken {
312        region_id: RegionId,
313        file_region_id: RegionId,
314        file_id: FileId,
315        #[snafu(implicit)]
316        location: Location,
317    },
318
319    #[snafu(display("Old manifest missing for region {}", region_id))]
320    MissingOldManifest {
321        region_id: RegionId,
322        #[snafu(implicit)]
323        location: Location,
324    },
325
326    #[snafu(display("New manifest missing for region {}", region_id))]
327    MissingNewManifest {
328        region_id: RegionId,
329        #[snafu(implicit)]
330        location: Location,
331    },
332
333    #[snafu(display("Manifest missing for region {}", region_id))]
334    MissingManifest {
335        region_id: RegionId,
336        #[snafu(implicit)]
337        location: Location,
338    },
339
340    #[snafu(display("File consistency check failed for file {}: {}", file_id, reason))]
341    InconsistentFile {
342        file_id: FileId,
343        reason: String,
344        #[snafu(implicit)]
345        location: Location,
346    },
347
348    #[snafu(display("Files lost during remapping: old={}, new={}", old_count, new_count))]
349    FilesLost {
350        old_count: usize,
351        new_count: usize,
352        #[snafu(implicit)]
353        location: Location,
354    },
355
356    #[snafu(display("No old manifests provided (need at least one for template)"))]
357    NoOldManifests {
358        #[snafu(implicit)]
359        location: Location,
360    },
361
362    #[snafu(display("Failed to fetch manifests"))]
363    FetchManifests {
364        #[snafu(implicit)]
365        location: Location,
366        source: BoxedError,
367    },
368
369    #[snafu(display("Partition expression missing for region {}", region_id))]
370    MissingPartitionExpr {
371        region_id: RegionId,
372        #[snafu(implicit)]
373        location: Location,
374    },
375
376    #[snafu(display("Failed to serialize partition expression: {}", source))]
377    SerializePartitionExpr {
378        #[snafu(source)]
379        source: PartitionError,
380        #[snafu(implicit)]
381        location: Location,
382    },
383
384    #[snafu(display(
385        "Failed to convert ConcreteDataType to ColumnDataType, reason: {}",
386        reason
387    ))]
388    ConvertColumnDataType {
389        reason: String,
390        source: api::error::Error,
391        #[snafu(implicit)]
392        location: Location,
393    },
394
395    /// An error type to indicate that schema is changed and we need
396    /// to fill default values again.
397    #[snafu(display("Need to fill default value for region {}", region_id))]
398    FillDefault {
399        region_id: RegionId,
400        // The error is for internal use so we don't need a location.
401    },
402
403    #[snafu(display(
404        "Failed to create default value for column {} of region {}",
405        column,
406        region_id
407    ))]
408    CreateDefault {
409        region_id: RegionId,
410        column: String,
411        source: datatypes::Error,
412        #[snafu(implicit)]
413        location: Location,
414    },
415
416    #[snafu(display("Failed to build entry, region_id: {}", region_id))]
417    BuildEntry {
418        region_id: RegionId,
419        #[snafu(implicit)]
420        location: Location,
421        source: BoxedError,
422    },
423
424    #[snafu(display("Failed to write WAL"))]
425    WriteWal {
426        #[snafu(implicit)]
427        location: Location,
428        source: BoxedError,
429    },
430
431    #[snafu(display("Failed to read WAL, provider: {}", provider))]
432    ReadWal {
433        provider: Provider,
434        #[snafu(implicit)]
435        location: Location,
436        source: BoxedError,
437    },
438
439    #[snafu(display("Failed to decode WAL entry, region_id: {}", region_id))]
440    DecodeWal {
441        region_id: RegionId,
442        #[snafu(implicit)]
443        location: Location,
444        #[snafu(source)]
445        error: DecodeError,
446    },
447
448    #[snafu(display("Failed to delete WAL, region_id: {}", region_id))]
449    DeleteWal {
450        region_id: RegionId,
451        #[snafu(implicit)]
452        location: Location,
453        source: BoxedError,
454    },
455
456    #[snafu(display("Failed to wait for the WAL to be durable, region_id: {}", region_id))]
457    WaitWalDurable {
458        region_id: RegionId,
459        #[snafu(implicit)]
460        location: Location,
461        source: BoxedError,
462    },
463
464    // Shared error for each writer in the write group.
465    #[snafu(display("Failed to write region"))]
466    WriteGroup { source: Arc<Error> },
467
468    #[snafu(display("Invalid parquet SST file {}, reason: {}", file, reason))]
469    InvalidParquet {
470        file: String,
471        reason: String,
472        #[snafu(implicit)]
473        location: Location,
474    },
475
476    #[snafu(display("Invalid batch, {}", reason))]
477    InvalidBatch {
478        reason: String,
479        #[snafu(implicit)]
480        location: Location,
481    },
482
483    #[snafu(display("Invalid arrow record batch, {}", reason))]
484    InvalidRecordBatch {
485        reason: String,
486        #[snafu(implicit)]
487        location: Location,
488    },
489
490    #[snafu(display("Invalid wal read request, {}", reason))]
491    InvalidWalReadRequest {
492        reason: String,
493        #[snafu(implicit)]
494        location: Location,
495    },
496
497    #[snafu(display("Failed to convert array to vector"))]
498    ConvertVector {
499        #[snafu(implicit)]
500        location: Location,
501        source: datatypes::error::Error,
502    },
503
504    #[snafu(display("Failed to compute arrow arrays"))]
505    ComputeArrow {
506        #[snafu(implicit)]
507        location: Location,
508        #[snafu(source)]
509        error: datatypes::arrow::error::ArrowError,
510    },
511
512    #[snafu(display("Failed to evaluate partition filter"))]
513    EvalPartitionFilter {
514        #[snafu(implicit)]
515        location: Location,
516        #[snafu(source)]
517        error: datafusion::error::DataFusionError,
518    },
519
520    #[snafu(display("Failed to merge candidate series"))]
521    MergeCandidateSeries {
522        #[snafu(implicit)]
523        location: Location,
524        #[snafu(source)]
525        error: datafusion::error::DataFusionError,
526    },
527
528    #[snafu(display("Failed to compute vector"))]
529    ComputeVector {
530        #[snafu(implicit)]
531        location: Location,
532        source: datatypes::error::Error,
533    },
534
535    #[snafu(display("Primary key length mismatch, expect: {}, actual: {}", expect, actual))]
536    PrimaryKeyLengthMismatch {
537        expect: usize,
538        actual: usize,
539        #[snafu(implicit)]
540        location: Location,
541    },
542
543    #[snafu(display("Invalid sender",))]
544    InvalidSender {
545        #[snafu(implicit)]
546        location: Location,
547    },
548
549    #[snafu(display("Invalid scheduler state"))]
550    InvalidSchedulerState {
551        #[snafu(implicit)]
552        location: Location,
553    },
554
555    #[snafu(display("Failed to stop scheduler"))]
556    StopScheduler {
557        #[snafu(source)]
558        error: JoinError,
559        #[snafu(implicit)]
560        location: Location,
561    },
562
563    #[snafu(display(
564        "Failed to batch delete SST files, region id: {}, file ids: {:?}",
565        region_id,
566        file_ids
567    ))]
568    DeleteSsts {
569        region_id: RegionId,
570        file_ids: Vec<FileId>,
571        #[snafu(source)]
572        error: object_store::Error,
573        #[snafu(implicit)]
574        location: Location,
575    },
576
577    #[snafu(display("Failed to delete index file, file id: {}", file_id))]
578    DeleteIndex {
579        file_id: FileId,
580        #[snafu(source)]
581        error: object_store::Error,
582        #[snafu(implicit)]
583        location: Location,
584    },
585
586    #[snafu(display("Failed to batch delete index files, file ids: {:?}", file_ids))]
587    DeleteIndexes {
588        file_ids: Vec<FileId>,
589        #[snafu(source)]
590        error: object_store::Error,
591        #[snafu(implicit)]
592        location: Location,
593    },
594
595    #[snafu(display("Failed to flush region {}", region_id))]
596    FlushRegion {
597        region_id: RegionId,
598        source: Arc<Error>,
599        #[snafu(implicit)]
600        location: Location,
601    },
602
603    #[snafu(display("Region {} is dropped", region_id))]
604    RegionDropped {
605        region_id: RegionId,
606        #[snafu(implicit)]
607        location: Location,
608    },
609
610    #[snafu(display("Region {} is closed", region_id))]
611    RegionClosed {
612        region_id: RegionId,
613        #[snafu(implicit)]
614        location: Location,
615    },
616
617    #[snafu(display(
618        "Stale compaction execution for region {}, the region may have been reopened, truncated or the compaction was superseded",
619        region_id
620    ))]
621    StaleCompactionExecution {
622        region_id: RegionId,
623        #[snafu(implicit)]
624        location: Location,
625    },
626
627    #[snafu(display("Region {} is truncated", region_id))]
628    RegionTruncated {
629        region_id: RegionId,
630        #[snafu(implicit)]
631        location: Location,
632    },
633
634    #[snafu(display(
635        "Engine write buffer is full, rejecting write requests of region {}",
636        region_id,
637    ))]
638    RejectWrite {
639        region_id: RegionId,
640        #[snafu(implicit)]
641        location: Location,
642    },
643
644    #[snafu(display("Region {} rejects writes while in staging", region_id))]
645    RejectWriteInStaging {
646        region_id: RegionId,
647        #[snafu(implicit)]
648        location: Location,
649    },
650
651    #[snafu(display("Failed to compact region {}", region_id))]
652    CompactRegion {
653        region_id: RegionId,
654        source: Arc<Error>,
655        #[snafu(implicit)]
656        location: Location,
657    },
658
659    #[snafu(display("Failed to edit region {}", region_id))]
660    EditRegion {
661        region_id: RegionId,
662        source: Arc<Error>,
663        #[snafu(implicit)]
664        location: Location,
665    },
666
667    #[snafu(display(
668        "Failed to compat readers for region {}, reason: {}",
669        region_id,
670        reason,
671    ))]
672    CompatReader {
673        region_id: RegionId,
674        reason: String,
675        #[snafu(implicit)]
676        location: Location,
677    },
678
679    #[snafu(display("Invalue region req"))]
680    InvalidRegionRequest {
681        source: store_api::metadata::MetadataError,
682        #[snafu(implicit)]
683        location: Location,
684    },
685
686    #[snafu(display(
687        "Region {} is in {:?} state, which does not permit manifest updates.",
688        region_id,
689        state
690    ))]
691    UpdateManifest {
692        region_id: RegionId,
693        state: RegionRoleState,
694        #[snafu(implicit)]
695        location: Location,
696    },
697
698    #[snafu(display("Region {} is in {:?} state, expect: {:?}", region_id, state, expect))]
699    RegionState {
700        region_id: RegionId,
701        state: RegionRoleState,
702        expect: RegionRoleState,
703        #[snafu(implicit)]
704        location: Location,
705    },
706
707    #[snafu(display(
708        "Partition expr version mismatch for region {}: request {}, expected {}",
709        region_id,
710        request_version,
711        expected_version
712    ))]
713    PartitionExprVersionMismatch {
714        region_id: RegionId,
715        request_version: u64,
716        expected_version: u64,
717        #[snafu(implicit)]
718        location: Location,
719    },
720
721    #[snafu(display("Invalid options"))]
722    JsonOptions {
723        #[snafu(source)]
724        error: serde_json::Error,
725        #[snafu(implicit)]
726        location: Location,
727    },
728
729    #[snafu(display(
730        "Empty region directory, region_id: {}, region_dir: {}",
731        region_id,
732        region_dir,
733    ))]
734    EmptyRegionDir {
735        region_id: RegionId,
736        region_dir: String,
737        #[snafu(implicit)]
738        location: Location,
739    },
740
741    #[snafu(display("Empty manifest directory, manifest_dir: {}", manifest_dir,))]
742    EmptyManifestDir {
743        manifest_dir: String,
744        #[snafu(implicit)]
745        location: Location,
746    },
747
748    #[snafu(display("Column not found, column: {column}"))]
749    ColumnNotFound {
750        column: String,
751        #[snafu(implicit)]
752        location: Location,
753    },
754
755    #[snafu(display("Failed to build index applier"))]
756    BuildIndexApplier {
757        source: index::inverted_index::error::Error,
758        #[snafu(implicit)]
759        location: Location,
760    },
761
762    #[snafu(display("Failed to build index asynchronously in region {}", region_id))]
763    BuildIndexAsync {
764        region_id: RegionId,
765        source: Arc<Error>,
766        #[snafu(implicit)]
767        location: Location,
768    },
769
770    #[snafu(display("Failed to convert value"))]
771    ConvertValue {
772        source: datatypes::error::Error,
773        #[snafu(implicit)]
774        location: Location,
775    },
776
777    #[snafu(display("Failed to apply inverted index"))]
778    ApplyInvertedIndex {
779        source: index::inverted_index::error::Error,
780        #[snafu(implicit)]
781        location: Location,
782    },
783
784    #[snafu(display("Failed to apply bloom filter index"))]
785    ApplyBloomFilterIndex {
786        source: index::bloom_filter::error::Error,
787        #[snafu(implicit)]
788        location: Location,
789    },
790
791    #[snafu(display("Failed to push index value"))]
792    PushIndexValue {
793        source: index::inverted_index::error::Error,
794        #[snafu(implicit)]
795        location: Location,
796    },
797
798    #[snafu(display("Failed to write index completely"))]
799    IndexFinish {
800        source: index::inverted_index::error::Error,
801        #[snafu(implicit)]
802        location: Location,
803    },
804
805    #[snafu(display("Operate on aborted index"))]
806    OperateAbortedIndex {
807        #[snafu(implicit)]
808        location: Location,
809    },
810
811    #[snafu(display("Failed to read puffin blob"))]
812    PuffinReadBlob {
813        source: puffin::error::Error,
814        #[snafu(implicit)]
815        location: Location,
816    },
817
818    #[snafu(display("Failed to add blob to puffin file"))]
819    PuffinAddBlob {
820        source: puffin::error::Error,
821        #[snafu(implicit)]
822        location: Location,
823    },
824
825    #[snafu(display("Failed to clean dir {dir}"))]
826    CleanDir {
827        dir: String,
828        #[snafu(source)]
829        error: std::io::Error,
830        #[snafu(implicit)]
831        location: Location,
832    },
833
834    #[snafu(display("Invalid config, {reason}"))]
835    InvalidConfig {
836        reason: String,
837        #[snafu(implicit)]
838        location: Location,
839    },
840
841    #[snafu(display(
842        "Stale log entry found during replay, region: {}, flushed: {}, replayed: {}",
843        region_id,
844        flushed_entry_id,
845        unexpected_entry_id
846    ))]
847    StaleLogEntry {
848        region_id: RegionId,
849        flushed_entry_id: u64,
850        unexpected_entry_id: u64,
851    },
852
853    #[snafu(display(
854        "Failed to download file, region_id: {}, file_id: {}, file_type: {:?}",
855        region_id,
856        file_id,
857        file_type,
858    ))]
859    Download {
860        region_id: RegionId,
861        file_id: FileId,
862        file_type: FileType,
863        #[snafu(source)]
864        error: std::io::Error,
865        #[snafu(implicit)]
866        location: Location,
867    },
868
869    #[snafu(display(
870        "Failed to upload file, region_id: {}, file_id: {}, file_type: {:?}",
871        region_id,
872        file_id,
873        file_type,
874    ))]
875    Upload {
876        region_id: RegionId,
877        file_id: FileId,
878        file_type: FileType,
879        #[snafu(source)]
880        error: std::io::Error,
881        #[snafu(implicit)]
882        location: Location,
883    },
884
885    #[snafu(display("Failed to create directory {}", dir))]
886    CreateDir {
887        dir: String,
888        #[snafu(source)]
889        error: std::io::Error,
890    },
891
892    #[snafu(display("Record batch error"))]
893    RecordBatch {
894        source: common_recordbatch::error::Error,
895        #[snafu(implicit)]
896        location: Location,
897    },
898
899    #[snafu(display("BiErrors, first: {first}, second: {second}"))]
900    BiErrors {
901        first: Box<Error>,
902        second: Box<Error>,
903        #[snafu(implicit)]
904        location: Location,
905    },
906
907    #[snafu(display("Encode null value"))]
908    IndexEncodeNull {
909        #[snafu(implicit)]
910        location: Location,
911    },
912
913    #[snafu(display("Failed to encode memtable to Parquet bytes"))]
914    EncodeMemtable {
915        #[snafu(source)]
916        error: parquet::errors::ParquetError,
917        #[snafu(implicit)]
918        location: Location,
919    },
920
921    #[snafu(display("Partition {} out of range, {} in total", given, all))]
922    PartitionOutOfRange {
923        given: usize,
924        all: usize,
925        #[snafu(implicit)]
926        location: Location,
927    },
928
929    #[snafu(display("Failed to iter data part"))]
930    ReadDataPart {
931        #[snafu(implicit)]
932        location: Location,
933        #[snafu(source)]
934        error: parquet::errors::ParquetError,
935    },
936
937    #[snafu(display("Failed to read row group in memtable"))]
938    DecodeArrowRowGroup {
939        #[snafu(source)]
940        error: ArrowError,
941        #[snafu(implicit)]
942        location: Location,
943    },
944
945    #[snafu(display("Invalid region options, {}", reason))]
946    InvalidRegionOptions {
947        reason: String,
948        #[snafu(implicit)]
949        location: Location,
950    },
951
952    #[snafu(display("checksum mismatch (actual: {}, expected: {})", actual, expected))]
953    ChecksumMismatch { actual: u32, expected: u32 },
954
955    #[snafu(display(
956        "No checkpoint found, region: {}, last_version: {}",
957        region_id,
958        last_version
959    ))]
960    NoCheckpoint {
961        region_id: RegionId,
962        last_version: ManifestVersion,
963        #[snafu(implicit)]
964        location: Location,
965    },
966
967    #[snafu(display(
968        "No manifests found in range: [{}..{}), region: {}, last_version: {}",
969        start_version,
970        end_version,
971        region_id,
972        last_version
973    ))]
974    NoManifests {
975        region_id: RegionId,
976        start_version: ManifestVersion,
977        end_version: ManifestVersion,
978        last_version: ManifestVersion,
979        #[snafu(implicit)]
980        location: Location,
981    },
982
983    #[snafu(display(
984        "Failed to install manifest to {}, region: {}, available manifest version: {}, last version: {}",
985        target_version,
986        region_id,
987        available_version,
988        last_version
989    ))]
990    InstallManifestTo {
991        region_id: RegionId,
992        target_version: ManifestVersion,
993        available_version: ManifestVersion,
994        #[snafu(implicit)]
995        location: Location,
996        last_version: ManifestVersion,
997    },
998
999    #[snafu(display("Region {} is stopped", region_id))]
1000    RegionStopped {
1001        region_id: RegionId,
1002        #[snafu(implicit)]
1003        location: Location,
1004    },
1005
1006    #[snafu(display(
1007        "Time range predicate overflows, timestamp: {:?}, target unit: {}",
1008        timestamp,
1009        unit
1010    ))]
1011    TimeRangePredicateOverflow {
1012        timestamp: Timestamp,
1013        unit: TimeUnit,
1014        #[snafu(implicit)]
1015        location: Location,
1016    },
1017
1018    #[snafu(display("Failed to open region"))]
1019    OpenRegion {
1020        #[snafu(implicit)]
1021        location: Location,
1022        source: Arc<Error>,
1023    },
1024
1025    #[snafu(display(
1026        "Region {} does not satisfy requirement '{}': {}",
1027        region_id,
1028        requirement,
1029        reason
1030    ))]
1031    RegionRequirement {
1032        region_id: RegionId,
1033        requirement: &'static str,
1034        reason: &'static str,
1035        #[snafu(implicit)]
1036        location: Location,
1037    },
1038
1039    #[snafu(display("Failed to parse job id"))]
1040    ParseJobId {
1041        #[snafu(implicit)]
1042        location: Location,
1043        #[snafu(source)]
1044        error: uuid::Error,
1045    },
1046
1047    #[snafu(display("Operation is not supported: {}", err_msg))]
1048    UnsupportedOperation {
1049        err_msg: String,
1050        #[snafu(implicit)]
1051        location: Location,
1052    },
1053
1054    #[snafu(display(
1055        "Failed to remotely compact region {} by job {:?} due to {}",
1056        region_id,
1057        job_id,
1058        reason
1059    ))]
1060    RemoteCompaction {
1061        region_id: RegionId,
1062        job_id: Option<JobId>,
1063        reason: String,
1064        #[snafu(implicit)]
1065        location: Location,
1066    },
1067
1068    #[snafu(display("Failed to initialize puffin stager"))]
1069    PuffinInitStager {
1070        source: puffin::error::Error,
1071        #[snafu(implicit)]
1072        location: Location,
1073    },
1074
1075    #[snafu(display("Failed to purge puffin stager"))]
1076    PuffinPurgeStager {
1077        source: puffin::error::Error,
1078        #[snafu(implicit)]
1079        location: Location,
1080    },
1081
1082    #[snafu(display("Failed to build puffin reader"))]
1083    PuffinBuildReader {
1084        source: puffin::error::Error,
1085        #[snafu(implicit)]
1086        location: Location,
1087    },
1088
1089    #[snafu(display("Failed to retrieve index options from column metadata"))]
1090    IndexOptions {
1091        #[snafu(implicit)]
1092        location: Location,
1093        source: datatypes::error::Error,
1094        column_name: String,
1095    },
1096
1097    #[snafu(display("Failed to retrieve compression options of column {column_name}"))]
1098    CompressionOptions {
1099        #[snafu(implicit)]
1100        location: Location,
1101        source: datatypes::error::Error,
1102        column_name: String,
1103    },
1104
1105    #[snafu(display("Failed to create fulltext index creator"))]
1106    CreateFulltextCreator {
1107        source: index::fulltext_index::error::Error,
1108        #[snafu(implicit)]
1109        location: Location,
1110    },
1111
1112    #[snafu(display("Failed to cast vector of {from} to {to}"))]
1113    CastVector {
1114        #[snafu(implicit)]
1115        location: Location,
1116        from: ConcreteDataType,
1117        to: ConcreteDataType,
1118        source: datatypes::error::Error,
1119    },
1120
1121    #[snafu(display("Failed to push text to fulltext index"))]
1122    FulltextPushText {
1123        source: index::fulltext_index::error::Error,
1124        #[snafu(implicit)]
1125        location: Location,
1126    },
1127
1128    #[snafu(display("Failed to finalize fulltext index creator"))]
1129    FulltextFinish {
1130        source: index::fulltext_index::error::Error,
1131        #[snafu(implicit)]
1132        location: Location,
1133    },
1134
1135    #[snafu(display("Failed to apply fulltext index"))]
1136    ApplyFulltextIndex {
1137        source: index::fulltext_index::error::Error,
1138        #[snafu(implicit)]
1139        location: Location,
1140    },
1141
1142    #[snafu(display("SST file {} does not contain valid stats info", file_path))]
1143    StatsNotPresent {
1144        file_path: String,
1145        #[snafu(implicit)]
1146        location: Location,
1147    },
1148
1149    #[snafu(display("Failed to decode stats of file {}", file_path))]
1150    DecodeStats {
1151        file_path: String,
1152        #[snafu(implicit)]
1153        location: Location,
1154    },
1155
1156    #[snafu(display("Invalid SST primary key range: {reason}"))]
1157    InvalidPrimaryKeyRange {
1158        reason: String,
1159        #[snafu(implicit)]
1160        location: Location,
1161    },
1162
1163    #[snafu(display("Failed to decode SST primary key range {endpoint} endpoint"))]
1164    DecodePrimaryKeyRange {
1165        endpoint: &'static str,
1166        source: mito_codec::error::Error,
1167        #[snafu(implicit)]
1168        location: Location,
1169    },
1170
1171    #[snafu(display("Region {} is busy", region_id))]
1172    RegionBusy {
1173        region_id: RegionId,
1174        #[snafu(implicit)]
1175        location: Location,
1176    },
1177
1178    #[snafu(display("Failed to get schema metadata"))]
1179    GetSchemaMetadata {
1180        source: common_meta::error::Error,
1181        #[snafu(implicit)]
1182        location: Location,
1183    },
1184
1185    #[snafu(display("Timeout"))]
1186    Timeout {
1187        #[snafu(source)]
1188        error: Elapsed,
1189        #[snafu(implicit)]
1190        location: Location,
1191    },
1192
1193    #[snafu(display("Failed to read file metadata"))]
1194    Metadata {
1195        #[snafu(source)]
1196        error: std::io::Error,
1197        #[snafu(implicit)]
1198        location: Location,
1199    },
1200
1201    #[snafu(display("Failed to push value to bloom filter"))]
1202    PushBloomFilterValue {
1203        source: index::bloom_filter::error::Error,
1204        #[snafu(implicit)]
1205        location: Location,
1206    },
1207
1208    #[snafu(display("Failed to finish bloom filter"))]
1209    BloomFilterFinish {
1210        source: index::bloom_filter::error::Error,
1211        #[snafu(implicit)]
1212        location: Location,
1213    },
1214
1215    #[snafu(display("Manual compaction is override by following operations."))]
1216    ManualCompactionOverride {},
1217
1218    #[snafu(display("Manual compaction is already running for region {region_id}."))]
1219    ManualCompactionAlreadyRunning { region_id: RegionId },
1220
1221    #[snafu(display("Compaction is cancelled."))]
1222    CompactionCancelled {},
1223
1224    #[snafu(display("Flush is cancelled."))]
1225    FlushCancelled {},
1226
1227    #[snafu(display("Compaction memory exhausted for region {region_id} (policy: {policy})",))]
1228    CompactionMemoryExhausted {
1229        region_id: RegionId,
1230        policy: String,
1231        #[snafu(source)]
1232        source: common_memory_manager::Error,
1233        #[snafu(implicit)]
1234        location: Location,
1235    },
1236
1237    #[snafu(display(
1238        "Incompatible WAL provider change. This is typically caused by changing WAL provider in database config file without completely cleaning existing files. Global provider: {}, region provider: {}",
1239        global,
1240        region
1241    ))]
1242    IncompatibleWalProviderChange { global: String, region: String },
1243
1244    #[snafu(display("Expected mito manifest info"))]
1245    MitoManifestInfo {
1246        #[snafu(implicit)]
1247        location: Location,
1248    },
1249
1250    #[snafu(display("Failed to scan series"))]
1251    ScanSeries {
1252        #[snafu(implicit)]
1253        location: Location,
1254        source: Arc<Error>,
1255    },
1256
1257    #[snafu(display("Partition {} scan multiple times", partition))]
1258    ScanMultiTimes {
1259        partition: usize,
1260        #[snafu(implicit)]
1261        location: Location,
1262    },
1263
1264    #[snafu(display("Invalid partition expression: {}", expr))]
1265    InvalidPartitionExpr {
1266        expr: String,
1267        #[snafu(implicit)]
1268        location: Location,
1269        source: partition::error::Error,
1270    },
1271
1272    #[snafu(display("Failed to decode bulk wal entry"))]
1273    ConvertBulkWalEntry {
1274        #[snafu(implicit)]
1275        location: Location,
1276        source: common_grpc::Error,
1277    },
1278
1279    #[snafu(display("Failed to encode"))]
1280    Encode {
1281        #[snafu(implicit)]
1282        location: Location,
1283        source: mito_codec::error::Error,
1284    },
1285
1286    #[snafu(display("Failed to decode"))]
1287    Decode {
1288        #[snafu(implicit)]
1289        location: Location,
1290        source: mito_codec::error::Error,
1291    },
1292
1293    #[snafu(display("Unexpected: {reason}"))]
1294    Unexpected {
1295        reason: String,
1296        #[snafu(implicit)]
1297        location: Location,
1298    },
1299
1300    #[cfg(feature = "enterprise")]
1301    #[snafu(display("Failed to scan external range"))]
1302    ScanExternalRange {
1303        source: BoxedError,
1304        #[snafu(implicit)]
1305        location: Location,
1306    },
1307
1308    #[snafu(display(
1309        "Inconsistent timestamp column length, expect: {}, actual: {}",
1310        expected,
1311        actual
1312    ))]
1313    InconsistentTimestampLength {
1314        expected: usize,
1315        actual: usize,
1316        #[snafu(implicit)]
1317        location: Location,
1318    },
1319
1320    #[snafu(display(
1321        "Too many files to read concurrently: {}, max allowed: {}",
1322        actual,
1323        max
1324    ))]
1325    TooManyFilesToRead {
1326        actual: usize,
1327        max: usize,
1328        #[snafu(implicit)]
1329        location: Location,
1330    },
1331
1332    #[snafu(display("Duration out of range: {input:?}"))]
1333    DurationOutOfRange {
1334        input: std::time::Duration,
1335        #[snafu(source)]
1336        error: chrono::OutOfRangeError,
1337        #[snafu(implicit)]
1338        location: Location,
1339    },
1340
1341    #[snafu(display("GC job permit exhausted"))]
1342    TooManyGcJobs {
1343        #[snafu(implicit)]
1344        location: Location,
1345    },
1346
1347    #[snafu(display(
1348        "Staging partition expr mismatch, manifest: {:?}, request: {}",
1349        manifest_expr,
1350        request_expr
1351    ))]
1352    StagingPartitionExprMismatch {
1353        manifest_expr: Option<String>,
1354        request_expr: String,
1355        #[snafu(implicit)]
1356        location: Location,
1357    },
1358
1359    #[snafu(display(
1360        "Invalid source and target region, source: {}, target: {}",
1361        source_region_id,
1362        target_region_id
1363    ))]
1364    InvalidSourceAndTargetRegion {
1365        source_region_id: RegionId,
1366        target_region_id: RegionId,
1367        #[snafu(implicit)]
1368        location: Location,
1369    },
1370
1371    #[snafu(display("Failed to prune file"))]
1372    PruneFile {
1373        source: Arc<Error>,
1374        #[snafu(implicit)]
1375        location: Location,
1376    },
1377
1378    #[snafu(display("Failed to cast column"))]
1379    CastColumn {
1380        #[snafu(source)]
1381        error: datafusion::error::DataFusionError,
1382        #[snafu(implicit)]
1383        location: Location,
1384    },
1385
1386    #[snafu(display("Failed to generate Arrow schema from Parquet file: {}", file))]
1387    ParquetToArrowSchema {
1388        file: String,
1389        #[snafu(source)]
1390        error: parquet::errors::ParquetError,
1391        #[snafu(implicit)]
1392        location: Location,
1393    },
1394
1395    #[snafu(display(
1396        "Region {} is in {:?} state, expect: Writable, Staging or Downgrading",
1397        region_id,
1398        state
1399    ))]
1400    FlushableRegionState {
1401        region_id: RegionId,
1402        state: RegionRoleState,
1403        #[snafu(implicit)]
1404        location: Location,
1405    },
1406}
1407
1408pub type Result<T, E = Error> = std::result::Result<T, E>;
1409
1410impl Error {
1411    /// Returns true if we need to fill default value for a region.
1412    pub(crate) fn is_fill_default(&self) -> bool {
1413        matches!(self, Error::FillDefault { .. })
1414    }
1415
1416    /// Returns true if the file is not found on the object store.
1417    pub(crate) fn is_object_not_found(&self) -> bool {
1418        match self {
1419            Error::OpenDal { error, .. } => error.kind() == ErrorKind::NotFound,
1420            Error::ManifestDeltaNotFound { .. } => true,
1421            _ => false,
1422        }
1423    }
1424
1425    /// Returns whether a failed manifest update may have been persisted.
1426    ///
1427    /// Unknown errors are treated conservatively because deleting output SSTs after an ambiguous
1428    /// manifest write could leave the manifest referencing missing files.
1429    pub(crate) fn may_have_persisted_manifest_update(&self) -> bool {
1430        match self {
1431            Error::UpdateManifest { .. }
1432            | Error::RegionState { .. }
1433            | Error::RegionTruncated { .. }
1434            | Error::RegionStopped { .. }
1435            | Error::SerdeJson { .. }
1436            | Error::CompressObject { .. } => false,
1437            Error::OpenDal { error, .. } => !matches!(
1438                error.kind(),
1439                ErrorKind::Unsupported
1440                    | ErrorKind::ConfigInvalid
1441                    | ErrorKind::NotFound
1442                    | ErrorKind::PermissionDenied
1443                    | ErrorKind::IsADirectory
1444                    | ErrorKind::NotADirectory
1445                    | ErrorKind::AlreadyExists
1446                    | ErrorKind::RateLimited
1447                    | ErrorKind::IsSameFile
1448                    | ErrorKind::ConditionNotMatch
1449                    | ErrorKind::RangeNotSatisfied
1450            ),
1451            _ => true,
1452        }
1453    }
1454}
1455
1456impl ErrorExt for Error {
1457    fn status_code(&self) -> StatusCode {
1458        use Error::*;
1459
1460        match self {
1461            DataTypeMismatch { source, .. } => source.status_code(),
1462            OpenDal { .. } | ManifestDeltaNotFound { .. } | ReadParquet { .. } => {
1463                StatusCode::StorageUnavailable
1464            }
1465            WriteWal { source, .. }
1466            | ReadWal { source, .. }
1467            | DeleteWal { source, .. }
1468            | WaitWalDurable { source, .. } => source.status_code(),
1469            CompressObject { .. }
1470            | DecompressObject { .. }
1471            | SerdeJson { .. }
1472            | Utf8 { .. }
1473            | NewRecordBatch { .. }
1474            | RegionCorrupted { .. }
1475            | InconsistentFile { .. }
1476            | CreateDefault { .. }
1477            | InvalidParquet { .. }
1478            | OperateAbortedIndex { .. }
1479            | IndexEncodeNull { .. }
1480            | NoCheckpoint { .. }
1481            | NoManifests { .. }
1482            | FilesLost { .. }
1483            | InstallManifestTo { .. }
1484            | Unexpected { .. }
1485            | SerializeColumnMetadata { .. }
1486            | SerializeManifest { .. }
1487            | StagingPartitionExprMismatch { .. } => StatusCode::Unexpected,
1488
1489            RegionNotFound { .. } => StatusCode::RegionNotFound,
1490            ObjectStoreNotFound { .. }
1491            | InvalidScanIndex { .. }
1492            | InvalidMeta { .. }
1493            | InvalidRequest { .. }
1494            | PartitionExprVersionMismatch { .. }
1495            | FillDefault { .. }
1496            | ConvertColumnDataType { .. }
1497            | ColumnNotFound { .. }
1498            | InvalidMetadata { .. }
1499            | InvalidRegionOptions { .. }
1500            | InvalidWalReadRequest { .. }
1501            | PartitionOutOfRange { .. }
1502            | ParseJobId { .. }
1503            | DurationOutOfRange { .. }
1504            | MissingOldManifest { .. }
1505            | MissingNewManifest { .. }
1506            | MissingManifest { .. }
1507            | NoOldManifests { .. }
1508            | MissingPartitionExpr { .. }
1509            | SerializePartitionExpr { .. }
1510            | InvalidSourceAndTargetRegion { .. } => StatusCode::InvalidArguments,
1511
1512            IncrementalQueryStale { .. } | SnapshotFenceStale { .. } => StatusCode::RequestOutdated,
1513
1514            SequenceRangeUnsupported { .. } => StatusCode::Unsupported,
1515
1516            RegionSequenceDomainBroken { .. }
1517            | RegionMetadataNotFound { .. }
1518            | Join { .. }
1519            | WorkerStopped { .. }
1520            | Recv { .. }
1521            | DecodeWal { .. }
1522            | ComputeArrow { .. }
1523            | EvalPartitionFilter { .. }
1524            | MergeCandidateSeries { .. }
1525            | BiErrors { .. }
1526            | StopScheduler { .. }
1527            | ComputeVector { .. }
1528            | EncodeMemtable { .. }
1529            | CreateDir { .. }
1530            | ReadDataPart { .. }
1531            | BuildEntry { .. }
1532            | Metadata { .. }
1533            | CastColumn { .. }
1534            | MitoManifestInfo { .. }
1535            | ParquetToArrowSchema { .. } => StatusCode::Internal,
1536
1537            FetchManifests { source, .. } => source.status_code(),
1538
1539            OpenRegion { source, .. } => source.status_code(),
1540
1541            WriteParquet { .. } => StatusCode::StorageUnavailable,
1542            WriteGroup { source, .. } => source.status_code(),
1543            InvalidBatch { .. } => StatusCode::InvalidArguments,
1544            InvalidRecordBatch { .. } => StatusCode::InvalidArguments,
1545            ConvertVector { source, .. } => source.status_code(),
1546
1547            PrimaryKeyLengthMismatch { .. } => StatusCode::InvalidArguments,
1548            InvalidSender { .. } => StatusCode::InvalidArguments,
1549            InvalidSchedulerState { .. } => StatusCode::InvalidArguments,
1550            RegionRequirement { .. } => StatusCode::InvalidArguments,
1551            DeleteSsts { .. } | DeleteIndex { .. } | DeleteIndexes { .. } => {
1552                StatusCode::StorageUnavailable
1553            }
1554            FlushRegion { source, .. } | BuildIndexAsync { source, .. } => source.status_code(),
1555            RegionDropped { .. } => StatusCode::Cancelled,
1556            RegionClosed { .. } => StatusCode::Cancelled,
1557            StaleCompactionExecution { .. } => StatusCode::Cancelled,
1558            RegionTruncated { .. } => StatusCode::Cancelled,
1559            RejectWrite { .. } | RejectWriteInStaging { .. } => StatusCode::StorageUnavailable,
1560            CompactRegion { source, .. } => source.status_code(),
1561            EditRegion { source, .. } => source.status_code(),
1562            CompatReader { .. } => StatusCode::Unexpected,
1563            InvalidRegionRequest { source, .. } => source.status_code(),
1564            RegionState { .. } | UpdateManifest { .. } => StatusCode::RegionNotReady,
1565            JsonOptions { .. } => StatusCode::InvalidArguments,
1566            EmptyRegionDir { .. } | EmptyManifestDir { .. } => StatusCode::RegionNotFound,
1567            ConvertValue { source, .. } => source.status_code(),
1568            ApplyBloomFilterIndex { source, .. } => source.status_code(),
1569            InvalidPartitionExpr { source, .. } => source.status_code(),
1570            BuildIndexApplier { source, .. }
1571            | PushIndexValue { source, .. }
1572            | ApplyInvertedIndex { source, .. }
1573            | IndexFinish { source, .. } => source.status_code(),
1574            PuffinReadBlob { source, .. }
1575            | PuffinAddBlob { source, .. }
1576            | PuffinInitStager { source, .. }
1577            | PuffinBuildReader { source, .. }
1578            | PuffinPurgeStager { source, .. } => source.status_code(),
1579            CleanDir { .. } => StatusCode::Unexpected,
1580            InvalidConfig { .. } => StatusCode::InvalidArguments,
1581            StaleLogEntry { .. }
1582            | InvalidNativeHistogramSubfield { .. }
1583            | InvalidNativeHistogramFieldId { .. } => StatusCode::Unexpected,
1584
1585            External { source, .. } => source.status_code(),
1586
1587            RecordBatch { source, .. } => source.status_code(),
1588
1589            Download { .. } | Upload { .. } => StatusCode::StorageUnavailable,
1590            ChecksumMismatch { .. } => StatusCode::Unexpected,
1591            RegionStopped { .. } => StatusCode::RegionNotReady,
1592            TimeRangePredicateOverflow { .. } => StatusCode::InvalidArguments,
1593            UnsupportedOperation { .. } => StatusCode::Unsupported,
1594            RemoteCompaction { .. } => StatusCode::Unexpected,
1595
1596            IndexOptions { source, .. } => source.status_code(),
1597            CompressionOptions { source, .. } => source.status_code(),
1598            CreateFulltextCreator { source, .. } => source.status_code(),
1599            CastVector { source, .. } => source.status_code(),
1600            FulltextPushText { source, .. }
1601            | FulltextFinish { source, .. }
1602            | ApplyFulltextIndex { source, .. } => source.status_code(),
1603            DecodeStats { .. }
1604            | StatsNotPresent { .. }
1605            | InvalidPrimaryKeyRange { .. }
1606            | DecodePrimaryKeyRange { .. } => StatusCode::Internal,
1607            RegionBusy { .. } => StatusCode::RegionBusy,
1608            GetSchemaMetadata { source, .. } => source.status_code(),
1609            Timeout { .. } => StatusCode::Cancelled,
1610
1611            DecodeArrowRowGroup { .. } => StatusCode::Internal,
1612
1613            PushBloomFilterValue { source, .. } | BloomFilterFinish { source, .. } => {
1614                source.status_code()
1615            }
1616
1617            ManualCompactionOverride {} | CompactionCancelled {} | FlushCancelled {} => {
1618                StatusCode::Cancelled
1619            }
1620
1621            // A concurrent manual compaction fails fast instead of being queued;
1622            // the conflict is reported to the caller, which decides whether to
1623            // issue a new request after the running one finishes.
1624            ManualCompactionAlreadyRunning { .. } => StatusCode::RegionBusy,
1625
1626            CompactionMemoryExhausted { source, .. } => source.status_code(),
1627
1628            IncompatibleWalProviderChange { .. } => StatusCode::InvalidArguments,
1629
1630            ScanSeries { source, .. } => source.status_code(),
1631
1632            ScanMultiTimes { .. } => StatusCode::InvalidArguments,
1633            ConvertBulkWalEntry { source, .. } => source.status_code(),
1634
1635            Encode { source, .. } | Decode { source, .. } => source.status_code(),
1636
1637            #[cfg(feature = "enterprise")]
1638            ScanExternalRange { source, .. } => source.status_code(),
1639
1640            InconsistentTimestampLength { .. } => StatusCode::InvalidArguments,
1641
1642            TooManyFilesToRead { .. } | TooManyGcJobs { .. } => StatusCode::RateLimited,
1643
1644            PruneFile { source, .. } => source.status_code(),
1645
1646            FlushableRegionState { .. } => StatusCode::RegionNotReady,
1647        }
1648    }
1649
1650    fn as_any(&self) -> &dyn Any {
1651        self
1652    }
1653
1654    fn retry_hint(&self) -> RetryHint {
1655        use Error::*;
1656
1657        match self {
1658            PartitionExprVersionMismatch { .. }
1659            | RegionNotFound { .. }
1660            | RejectWriteInStaging { .. }
1661            | RegionState {
1662                state: RegionRoleState::Follower,
1663                ..
1664            } => RetryHint::RetryAfterRouteRefresh,
1665            ReadParquet { .. }
1666            | WriteParquet { .. }
1667            | RejectWrite { .. }
1668            | Download { .. }
1669            | Upload { .. }
1670            | RegionState { .. }
1671            | UpdateManifest { .. }
1672            | RegionStopped { .. }
1673            | RegionBusy { .. }
1674            | ManualCompactionAlreadyRunning { .. }
1675            | FlushableRegionState { .. }
1676            | ManifestDeltaNotFound { .. } => RetryHint::Retryable,
1677
1678            OpenDal { error, .. }
1679            | DeleteSsts { error, .. }
1680            | DeleteIndex { error, .. }
1681            | DeleteIndexes { error, .. } => retry_hint_from_opendal_error(error),
1682
1683            WriteWal { source, .. }
1684            | ReadWal { source, .. }
1685            | DeleteWal { source, .. }
1686            | WaitWalDurable { source, .. }
1687            | FetchManifests { source, .. }
1688            | External { source, .. } => source.retry_hint(),
1689
1690            OpenRegion { source, .. }
1691            | WriteGroup { source, .. }
1692            | FlushRegion { source, .. }
1693            | BuildIndexAsync { source, .. }
1694            | CompactRegion { source, .. }
1695            | EditRegion { source, .. }
1696            | ScanSeries { source, .. }
1697            | PruneFile { source, .. } => source.retry_hint(),
1698
1699            DataTypeMismatch { source, .. }
1700            | ConvertVector { source, .. }
1701            | ConvertValue { source, .. }
1702            | IndexOptions { source, .. }
1703            | CompressionOptions { source, .. }
1704            | CastVector { source, .. } => source.retry_hint(),
1705
1706            BuildIndexApplier { source, .. }
1707            | PushIndexValue { source, .. }
1708            | ApplyInvertedIndex { source, .. }
1709            | IndexFinish { source, .. } => source.retry_hint(),
1710
1711            ApplyBloomFilterIndex { source, .. }
1712            | PushBloomFilterValue { source, .. }
1713            | BloomFilterFinish { source, .. } => source.retry_hint(),
1714
1715            PuffinReadBlob { source, .. }
1716            | PuffinAddBlob { source, .. }
1717            | PuffinInitStager { source, .. }
1718            | PuffinBuildReader { source, .. }
1719            | PuffinPurgeStager { source, .. } => source.retry_hint(),
1720
1721            CreateFulltextCreator { source, .. }
1722            | FulltextPushText { source, .. }
1723            | FulltextFinish { source, .. }
1724            | ApplyFulltextIndex { source, .. } => source.retry_hint(),
1725
1726            InvalidRegionRequest { source, .. } => source.retry_hint(),
1727            InvalidPartitionExpr { source, .. } => source.retry_hint(),
1728            RecordBatch { source, .. } => source.retry_hint(),
1729            GetSchemaMetadata { source, .. } => source.retry_hint(),
1730            CompactionMemoryExhausted { source, .. } => source.retry_hint(),
1731            ConvertBulkWalEntry { source, .. } => source.retry_hint(),
1732            Encode { source, .. } | Decode { source, .. } => source.retry_hint(),
1733
1734            #[cfg(feature = "enterprise")]
1735            ScanExternalRange { source, .. } => source.retry_hint(),
1736
1737            _ => RetryHint::NonRetryable,
1738        }
1739    }
1740}
1741
1742#[cfg(test)]
1743mod tests {
1744    use snafu::IntoError;
1745
1746    use super::*;
1747    use crate::region::RegionLeaderState;
1748
1749    #[test]
1750    fn test_partition_version_mismatch_retry_hint() {
1751        let error = PartitionExprVersionMismatchSnafu {
1752            region_id: RegionId::new(1, 1),
1753            request_version: 1u64,
1754            expected_version: 2u64,
1755        }
1756        .build();
1757        assert_eq!(error.status_code(), StatusCode::InvalidArguments);
1758        assert_eq!(error.retry_hint(), RetryHint::RetryAfterRouteRefresh);
1759    }
1760
1761    #[test]
1762    fn test_region_state_retry_hint() {
1763        let region_id = RegionId::new(1, 1);
1764        let writable = RegionRoleState::Leader(RegionLeaderState::Writable);
1765        let follower_error = RegionStateSnafu {
1766            region_id,
1767            state: RegionRoleState::Follower,
1768            expect: writable,
1769        }
1770        .build();
1771        assert_eq!(
1772            follower_error.retry_hint(),
1773            RetryHint::RetryAfterRouteRefresh
1774        );
1775        assert_eq!(follower_error.status_code(), StatusCode::RegionNotReady);
1776        let leader_error = RegionStateSnafu {
1777            region_id,
1778            state: writable,
1779            expect: RegionRoleState::Follower,
1780        }
1781        .build();
1782        assert_eq!(leader_error.retry_hint(), RetryHint::Retryable);
1783        for (error, status, hint) in [
1784            (
1785                RejectWriteSnafu { region_id }.build(),
1786                StatusCode::StorageUnavailable,
1787                RetryHint::Retryable,
1788            ),
1789            (
1790                RejectWriteInStagingSnafu { region_id }.build(),
1791                StatusCode::StorageUnavailable,
1792                RetryHint::RetryAfterRouteRefresh,
1793            ),
1794            (
1795                RegionNotFoundSnafu { region_id }.build(),
1796                StatusCode::RegionNotFound,
1797                RetryHint::RetryAfterRouteRefresh,
1798            ),
1799        ] {
1800            assert_eq!(error.status_code(), status);
1801            assert_eq!(error.retry_hint(), hint);
1802        }
1803    }
1804
1805    #[test]
1806    fn test_manifest_update_persistence() {
1807        let rejected_kinds = [
1808            ErrorKind::Unsupported,
1809            ErrorKind::ConfigInvalid,
1810            ErrorKind::NotFound,
1811            ErrorKind::PermissionDenied,
1812            ErrorKind::IsADirectory,
1813            ErrorKind::NotADirectory,
1814            ErrorKind::AlreadyExists,
1815            ErrorKind::RateLimited,
1816            ErrorKind::IsSameFile,
1817            ErrorKind::ConditionNotMatch,
1818            ErrorKind::RangeNotSatisfied,
1819        ];
1820        for kind in rejected_kinds {
1821            let error = OpenDalSnafu {}.into_error(object_store::Error::new(kind, "test"));
1822            assert!(
1823                !error.may_have_persisted_manifest_update(),
1824                "error kind {kind:?} should prove the manifest was not persisted"
1825            );
1826        }
1827
1828        let error =
1829            OpenDalSnafu {}.into_error(object_store::Error::new(ErrorKind::Unexpected, "test"));
1830        assert!(error.may_have_persisted_manifest_update());
1831
1832        let region_id = RegionId::new(1, 1);
1833        let error = RegionTruncatedSnafu { region_id }.build();
1834        assert!(!error.may_have_persisted_manifest_update());
1835
1836        // This can be raised while applying an edit after its manifest file was saved.
1837        let error = RegionMetadataNotFoundSnafu {}.build();
1838        assert!(error.may_have_persisted_manifest_update());
1839    }
1840}
mito2/error.rs

mito2/
error.rs
