mito2/
error.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::any::Any;
16use std::sync::Arc;
17
18use common_datasource::compression::CompressionType;
19use common_error::ext::{BoxedError, ErrorExt, RetryHint};
20use common_error::status_code::StatusCode;
21use common_macro::stack_trace_debug;
22use common_memory_manager;
23use common_runtime::JoinError;
24use common_time::Timestamp;
25use common_time::timestamp::TimeUnit;
26use datatypes::arrow::error::ArrowError;
27use datatypes::prelude::ConcreteDataType;
28use object_store::ErrorKind;
29use object_store::error::retry_hint_from_opendal_error;
30use partition::error::Error as PartitionError;
31use prost::DecodeError;
32use snafu::{Location, Snafu};
33use store_api::ManifestVersion;
34use store_api::logstore::provider::Provider;
35use store_api::storage::{FileId, RegionId};
36use tokio::time::error::Elapsed;
37
38use crate::cache::file_cache::FileType;
39use crate::region::RegionRoleState;
40use crate::schedule::remote_job_scheduler::JobId;
41use crate::worker::WorkerId;
42
43#[derive(Snafu)]
44#[snafu(visibility(pub))]
45#[stack_trace_debug]
46pub enum Error {
47    #[snafu(display("Unexpected data type"))]
48    DataTypeMismatch {
49        source: datatypes::error::Error,
50        #[snafu(implicit)]
51        location: Location,
52    },
53
54    #[snafu(display("External error, context: {}", context))]
55    External {
56        source: BoxedError,
57        context: String,
58        #[snafu(implicit)]
59        location: Location,
60    },
61
62    #[snafu(display("OpenDAL operator failed"))]
63    OpenDal {
64        #[snafu(implicit)]
65        location: Location,
66        #[snafu(source)]
67        error: object_store::Error,
68    },
69
70    #[snafu(display(
71        "Manifest delta {} disappeared after it was listed, path: {}",
72        version,
73        path
74    ))]
75    ManifestDeltaNotFound {
76        version: ManifestVersion,
77        path: String,
78        #[snafu(source)]
79        error: object_store::Error,
80        #[snafu(implicit)]
81        location: Location,
82    },
83
84    #[snafu(display("Fail to compress object by {}, path: {}", compress_type, path))]
85    CompressObject {
86        compress_type: CompressionType,
87        path: String,
88        #[snafu(source)]
89        error: std::io::Error,
90    },
91
92    #[snafu(display("Fail to decompress object by {}, path: {}", compress_type, path))]
93    DecompressObject {
94        compress_type: CompressionType,
95        path: String,
96        #[snafu(source)]
97        error: std::io::Error,
98    },
99
100    #[snafu(display("Failed to ser/de json object"))]
101    SerdeJson {
102        #[snafu(implicit)]
103        location: Location,
104        #[snafu(source)]
105        error: serde_json::Error,
106    },
107
108    #[snafu(display("Failed to serialize column metadata"))]
109    SerializeColumnMetadata {
110        #[snafu(source)]
111        error: serde_json::Error,
112        #[snafu(implicit)]
113        location: Location,
114    },
115
116    #[snafu(display("Failed to serialize manifest, region_id: {}", region_id))]
117    SerializeManifest {
118        region_id: RegionId,
119        #[snafu(source)]
120        error: serde_json::Error,
121        #[snafu(implicit)]
122        location: Location,
123    },
124
125    #[snafu(display("Invalid scan index, start: {}, end: {}", start, end))]
126    InvalidScanIndex {
127        start: ManifestVersion,
128        end: ManifestVersion,
129        #[snafu(implicit)]
130        location: Location,
131    },
132
133    #[snafu(display("Invalid UTF-8 content"))]
134    Utf8 {
135        #[snafu(implicit)]
136        location: Location,
137        #[snafu(source)]
138        error: std::str::Utf8Error,
139    },
140
141    #[snafu(display("Cannot find RegionMetadata"))]
142    RegionMetadataNotFound {
143        #[snafu(implicit)]
144        location: Location,
145    },
146
147    #[snafu(display("Failed to join handle"))]
148    Join {
149        #[snafu(source)]
150        error: common_runtime::JoinError,
151        #[snafu(implicit)]
152        location: Location,
153    },
154
155    #[snafu(display("Worker {} is stopped", id))]
156    WorkerStopped {
157        id: WorkerId,
158        #[snafu(implicit)]
159        location: Location,
160    },
161
162    #[snafu(display("Failed to recv result"))]
163    Recv {
164        #[snafu(source)]
165        error: tokio::sync::oneshot::error::RecvError,
166        #[snafu(implicit)]
167        location: Location,
168    },
169
170    #[snafu(display("Invalid metadata, {}", reason))]
171    InvalidMeta {
172        reason: String,
173        #[snafu(implicit)]
174        location: Location,
175    },
176
177    #[snafu(display("Invalid region metadata"))]
178    InvalidMetadata {
179        source: store_api::metadata::MetadataError,
180        #[snafu(implicit)]
181        location: Location,
182    },
183
184    #[snafu(display("Failed to create RecordBatch from vectors"))]
185    NewRecordBatch {
186        #[snafu(implicit)]
187        location: Location,
188        #[snafu(source)]
189        error: ArrowError,
190    },
191
192    #[snafu(display("Failed to read parquet file, path: {}", path))]
193    ReadParquet {
194        path: String,
195        #[snafu(source)]
196        error: parquet::errors::ParquetError,
197        #[snafu(implicit)]
198        location: Location,
199    },
200
201    #[snafu(display("Failed to write parquet file"))]
202    WriteParquet {
203        #[snafu(source)]
204        error: parquet::errors::ParquetError,
205        #[snafu(implicit)]
206        location: Location,
207    },
208
209    #[snafu(display(
210        "Cannot assign a stable field id to native histogram sub-field '{}' of column id {} (unknown sub-field name or derived id overflows i32)",
211        field_name,
212        column_id
213    ))]
214    InvalidNativeHistogramSubfield {
215        column_id: i32,
216        field_name: String,
217        #[snafu(implicit)]
218        location: Location,
219    },
220
221    #[snafu(display(
222        "Native histogram column '{}' has no usable PARQUET:field_id to namespace its sub-field ids (missing, malformed, or exceeds i32::MAX)",
223        field_name
224    ))]
225    InvalidNativeHistogramFieldId {
226        field_name: String,
227        #[snafu(implicit)]
228        location: Location,
229    },
230
231    #[snafu(display("Region {} not found", region_id))]
232    RegionNotFound {
233        region_id: RegionId,
234        #[snafu(implicit)]
235        location: Location,
236    },
237
238    #[snafu(display("Object store not found: {}", object_store))]
239    ObjectStoreNotFound {
240        object_store: String,
241        #[snafu(implicit)]
242        location: Location,
243    },
244
245    #[snafu(display("Region {} is corrupted, reason: {}", region_id, reason))]
246    RegionCorrupted {
247        region_id: RegionId,
248        reason: String,
249        #[snafu(implicit)]
250        location: Location,
251    },
252
253    #[snafu(display("Invalid request to region {}, reason: {}", region_id, reason))]
254    InvalidRequest {
255        region_id: RegionId,
256        reason: String,
257        #[snafu(implicit)]
258        location: Location,
259    },
260
261    #[snafu(display(
262        "STALE_CURSOR: incremental query stale, region: {}, given_seq: {}, min_readable_seq: {}, retry_hint: FALLBACK_FULL_RECOMPUTE",
263        region_id,
264        given_seq,
265        min_readable_seq
266    ))]
267    IncrementalQueryStale {
268        region_id: RegionId,
269        given_seq: u64,
270        min_readable_seq: u64,
271        #[snafu(implicit)]
272        location: Location,
273    },
274
275    #[snafu(display(
276        "STALE_SNAPSHOT_FENCE: snapshot upper bound stale, region: {}, given_seq: {}, min_enforceable_seq: {}, retry_hint: REBIND_SNAPSHOT_FENCE",
277        region_id,
278        given_seq,
279        min_enforceable_seq
280    ))]
281    SnapshotFenceStale {
282        region_id: RegionId,
283        given_seq: u64,
284        min_enforceable_seq: u64,
285        #[snafu(implicit)]
286        location: Location,
287    },
288
289    #[snafu(display(
290        "SEQUENCE_RANGE_UNSUPPORTED: exact sequence-range read unsupported, region: {}, min_seq: {}, max_seq: {}, reason: {}, retry_hint: FALLBACK_MEMTABLE_ONLY_OR_FULL_RECOMPUTE",
291        region_id,
292        min_seq,
293        max_seq,
294        reason
295    ))]
296    SequenceRangeUnsupported {
297        region_id: RegionId,
298        min_seq: u64,
299        max_seq: u64,
300        reason: String,
301        #[snafu(implicit)]
302        location: Location,
303    },
304
305    #[snafu(display(
306        "region {} is unusable for sequence reads: file {} declares region {}",
307        region_id,
308        file_id,
309        file_region_id
310    ))]
311    RegionSequenceDomainBroken {
312        region_id: RegionId,
313        file_region_id: RegionId,
314        file_id: FileId,
315        #[snafu(implicit)]
316        location: Location,
317    },
318
319    #[snafu(display("Old manifest missing for region {}", region_id))]
320    MissingOldManifest {
321        region_id: RegionId,
322        #[snafu(implicit)]
323        location: Location,
324    },
325
326    #[snafu(display("New manifest missing for region {}", region_id))]
327    MissingNewManifest {
328        region_id: RegionId,
329        #[snafu(implicit)]
330        location: Location,
331    },
332
333    #[snafu(display("Manifest missing for region {}", region_id))]
334    MissingManifest {
335        region_id: RegionId,
336        #[snafu(implicit)]
337        location: Location,
338    },
339
340    #[snafu(display("File consistency check failed for file {}: {}", file_id, reason))]
341    InconsistentFile {
342        file_id: FileId,
343        reason: String,
344        #[snafu(implicit)]
345        location: Location,
346    },
347
348    #[snafu(display("Files lost during remapping: old={}, new={}", old_count, new_count))]
349    FilesLost {
350        old_count: usize,
351        new_count: usize,
352        #[snafu(implicit)]
353        location: Location,
354    },
355
356    #[snafu(display("No old manifests provided (need at least one for template)"))]
357    NoOldManifests {
358        #[snafu(implicit)]
359        location: Location,
360    },
361
362    #[snafu(display("Failed to fetch manifests"))]
363    FetchManifests {
364        #[snafu(implicit)]
365        location: Location,
366        source: BoxedError,
367    },
368
369    #[snafu(display("Partition expression missing for region {}", region_id))]
370    MissingPartitionExpr {
371        region_id: RegionId,
372        #[snafu(implicit)]
373        location: Location,
374    },
375
376    #[snafu(display("Failed to serialize partition expression: {}", source))]
377    SerializePartitionExpr {
378        #[snafu(source)]
379        source: PartitionError,
380        #[snafu(implicit)]
381        location: Location,
382    },
383
384    #[snafu(display(
385        "Failed to convert ConcreteDataType to ColumnDataType, reason: {}",
386        reason
387    ))]
388    ConvertColumnDataType {
389        reason: String,
390        source: api::error::Error,
391        #[snafu(implicit)]
392        location: Location,
393    },
394
395    /// An error type to indicate that schema is changed and we need
396    /// to fill default values again.
397    #[snafu(display("Need to fill default value for region {}", region_id))]
398    FillDefault {
399        region_id: RegionId,
400        // The error is for internal use so we don't need a location.
401    },
402
403    #[snafu(display(
404        "Failed to create default value for column {} of region {}",
405        column,
406        region_id
407    ))]
408    CreateDefault {
409        region_id: RegionId,
410        column: String,
411        source: datatypes::Error,
412        #[snafu(implicit)]
413        location: Location,
414    },
415
416    #[snafu(display("Failed to build entry, region_id: {}", region_id))]
417    BuildEntry {
418        region_id: RegionId,
419        #[snafu(implicit)]
420        location: Location,
421        source: BoxedError,
422    },
423
424    #[snafu(display("Failed to write WAL"))]
425    WriteWal {
426        #[snafu(implicit)]
427        location: Location,
428        source: BoxedError,
429    },
430
431    #[snafu(display("Failed to read WAL, provider: {}", provider))]
432    ReadWal {
433        provider: Provider,
434        #[snafu(implicit)]
435        location: Location,
436        source: BoxedError,
437    },
438
439    #[snafu(display("Failed to decode WAL entry, region_id: {}", region_id))]
440    DecodeWal {
441        region_id: RegionId,
442        #[snafu(implicit)]
443        location: Location,
444        #[snafu(source)]
445        error: DecodeError,
446    },
447
448    #[snafu(display("Failed to delete WAL, region_id: {}", region_id))]
449    DeleteWal {
450        region_id: RegionId,
451        #[snafu(implicit)]
452        location: Location,
453        source: BoxedError,
454    },
455
456    #[snafu(display("Failed to wait for the WAL to be durable, region_id: {}", region_id))]
457    WaitWalDurable {
458        region_id: RegionId,
459        #[snafu(implicit)]
460        location: Location,
461        source: BoxedError,
462    },
463
464    #[snafu(display("Failed to check the WAL history, region_id: {}", region_id))]
465    CheckWalHistory {
466        region_id: RegionId,
467        #[snafu(implicit)]
468        location: Location,
469        source: BoxedError,
470    },
471
472    // Shared error for each writer in the write group.
473    #[snafu(display("Failed to write region"))]
474    WriteGroup { source: Arc<Error> },
475
476    #[snafu(display("Invalid parquet SST file {}, reason: {}", file, reason))]
477    InvalidParquet {
478        file: String,
479        reason: String,
480        #[snafu(implicit)]
481        location: Location,
482    },
483
484    #[snafu(display("Invalid batch, {}", reason))]
485    InvalidBatch {
486        reason: String,
487        #[snafu(implicit)]
488        location: Location,
489    },
490
491    #[snafu(display("Invalid arrow record batch, {}", reason))]
492    InvalidRecordBatch {
493        reason: String,
494        #[snafu(implicit)]
495        location: Location,
496    },
497
498    #[snafu(display("Invalid wal read request, {}", reason))]
499    InvalidWalReadRequest {
500        reason: String,
501        #[snafu(implicit)]
502        location: Location,
503    },
504
505    #[snafu(display("Failed to convert array to vector"))]
506    ConvertVector {
507        #[snafu(implicit)]
508        location: Location,
509        source: datatypes::error::Error,
510    },
511
512    #[snafu(display("Failed to compute arrow arrays"))]
513    ComputeArrow {
514        #[snafu(implicit)]
515        location: Location,
516        #[snafu(source)]
517        error: datatypes::arrow::error::ArrowError,
518    },
519
520    #[snafu(display("Failed to evaluate partition filter"))]
521    EvalPartitionFilter {
522        #[snafu(implicit)]
523        location: Location,
524        #[snafu(source)]
525        error: datafusion::error::DataFusionError,
526    },
527
528    #[snafu(display("Failed to merge candidate series"))]
529    MergeCandidateSeries {
530        #[snafu(implicit)]
531        location: Location,
532        #[snafu(source)]
533        error: datafusion::error::DataFusionError,
534    },
535
536    #[snafu(display("Failed to compute vector"))]
537    ComputeVector {
538        #[snafu(implicit)]
539        location: Location,
540        source: datatypes::error::Error,
541    },
542
543    #[snafu(display("Primary key length mismatch, expect: {}, actual: {}", expect, actual))]
544    PrimaryKeyLengthMismatch {
545        expect: usize,
546        actual: usize,
547        #[snafu(implicit)]
548        location: Location,
549    },
550
551    #[snafu(display("Invalid sender",))]
552    InvalidSender {
553        #[snafu(implicit)]
554        location: Location,
555    },
556
557    #[snafu(display("Invalid scheduler state"))]
558    InvalidSchedulerState {
559        #[snafu(implicit)]
560        location: Location,
561    },
562
563    #[snafu(display("Failed to stop scheduler"))]
564    StopScheduler {
565        #[snafu(source)]
566        error: JoinError,
567        #[snafu(implicit)]
568        location: Location,
569    },
570
571    #[snafu(display(
572        "Failed to batch delete SST files, region id: {}, file ids: {:?}",
573        region_id,
574        file_ids
575    ))]
576    DeleteSsts {
577        region_id: RegionId,
578        file_ids: Vec<FileId>,
579        #[snafu(source)]
580        error: object_store::Error,
581        #[snafu(implicit)]
582        location: Location,
583    },
584
585    #[snafu(display("Failed to delete index file, file id: {}", file_id))]
586    DeleteIndex {
587        file_id: FileId,
588        #[snafu(source)]
589        error: object_store::Error,
590        #[snafu(implicit)]
591        location: Location,
592    },
593
594    #[snafu(display("Failed to batch delete index files, file ids: {:?}", file_ids))]
595    DeleteIndexes {
596        file_ids: Vec<FileId>,
597        #[snafu(source)]
598        error: object_store::Error,
599        #[snafu(implicit)]
600        location: Location,
601    },
602
603    #[snafu(display("Failed to flush region {}", region_id))]
604    FlushRegion {
605        region_id: RegionId,
606        source: Arc<Error>,
607        #[snafu(implicit)]
608        location: Location,
609    },
610
611    #[snafu(display("Region {} is dropped", region_id))]
612    RegionDropped {
613        region_id: RegionId,
614        #[snafu(implicit)]
615        location: Location,
616    },
617
618    #[snafu(display("Region {} is closed", region_id))]
619    RegionClosed {
620        region_id: RegionId,
621        #[snafu(implicit)]
622        location: Location,
623    },
624
625    #[snafu(display(
626        "Stale compaction execution for region {}, the region may have been reopened, truncated or the compaction was superseded",
627        region_id
628    ))]
629    StaleCompactionExecution {
630        region_id: RegionId,
631        #[snafu(implicit)]
632        location: Location,
633    },
634
635    #[snafu(display("Region {} is truncated", region_id))]
636    RegionTruncated {
637        region_id: RegionId,
638        #[snafu(implicit)]
639        location: Location,
640    },
641
642    #[snafu(display(
643        "Engine write buffer is full, rejecting write requests of region {}",
644        region_id,
645    ))]
646    RejectWrite {
647        region_id: RegionId,
648        #[snafu(implicit)]
649        location: Location,
650    },
651
652    #[snafu(display("Region {} rejects writes while in staging", region_id))]
653    RejectWriteInStaging {
654        region_id: RegionId,
655        #[snafu(implicit)]
656        location: Location,
657    },
658
659    #[snafu(display("Failed to compact region {}", region_id))]
660    CompactRegion {
661        region_id: RegionId,
662        source: Arc<Error>,
663        #[snafu(implicit)]
664        location: Location,
665    },
666
667    #[snafu(display("Failed to edit region {}", region_id))]
668    EditRegion {
669        region_id: RegionId,
670        source: Arc<Error>,
671        #[snafu(implicit)]
672        location: Location,
673    },
674
675    #[snafu(display(
676        "Failed to compat readers for region {}, reason: {}",
677        region_id,
678        reason,
679    ))]
680    CompatReader {
681        region_id: RegionId,
682        reason: String,
683        #[snafu(implicit)]
684        location: Location,
685    },
686
687    #[snafu(display("Invalue region req"))]
688    InvalidRegionRequest {
689        source: store_api::metadata::MetadataError,
690        #[snafu(implicit)]
691        location: Location,
692    },
693
694    #[snafu(display(
695        "Region {} is in {:?} state, which does not permit manifest updates.",
696        region_id,
697        state
698    ))]
699    UpdateManifest {
700        region_id: RegionId,
701        state: RegionRoleState,
702        #[snafu(implicit)]
703        location: Location,
704    },
705
706    #[snafu(display("Region {} is in {:?} state, expect: {:?}", region_id, state, expect))]
707    RegionState {
708        region_id: RegionId,
709        state: RegionRoleState,
710        expect: RegionRoleState,
711        #[snafu(implicit)]
712        location: Location,
713    },
714
715    #[snafu(display(
716        "Partition expr version mismatch for region {}: request {}, expected {}",
717        region_id,
718        request_version,
719        expected_version
720    ))]
721    PartitionExprVersionMismatch {
722        region_id: RegionId,
723        request_version: u64,
724        expected_version: u64,
725        #[snafu(implicit)]
726        location: Location,
727    },
728
729    #[snafu(display("Invalid options"))]
730    JsonOptions {
731        #[snafu(source)]
732        error: serde_json::Error,
733        #[snafu(implicit)]
734        location: Location,
735    },
736
737    #[snafu(display(
738        "Empty region directory, region_id: {}, region_dir: {}",
739        region_id,
740        region_dir,
741    ))]
742    EmptyRegionDir {
743        region_id: RegionId,
744        region_dir: String,
745        #[snafu(implicit)]
746        location: Location,
747    },
748
749    #[snafu(display("Empty manifest directory, manifest_dir: {}", manifest_dir,))]
750    EmptyManifestDir {
751        manifest_dir: String,
752        #[snafu(implicit)]
753        location: Location,
754    },
755
756    #[snafu(display("Column not found, column: {column}"))]
757    ColumnNotFound {
758        column: String,
759        #[snafu(implicit)]
760        location: Location,
761    },
762
763    #[snafu(display("Failed to build index applier"))]
764    BuildIndexApplier {
765        source: index::inverted_index::error::Error,
766        #[snafu(implicit)]
767        location: Location,
768    },
769
770    #[snafu(display("Failed to build index asynchronously in region {}", region_id))]
771    BuildIndexAsync {
772        region_id: RegionId,
773        source: Arc<Error>,
774        #[snafu(implicit)]
775        location: Location,
776    },
777
778    #[snafu(display("Failed to convert value"))]
779    ConvertValue {
780        source: datatypes::error::Error,
781        #[snafu(implicit)]
782        location: Location,
783    },
784
785    #[snafu(display("Failed to apply inverted index"))]
786    ApplyInvertedIndex {
787        source: index::inverted_index::error::Error,
788        #[snafu(implicit)]
789        location: Location,
790    },
791
792    #[snafu(display("Failed to apply bloom filter index"))]
793    ApplyBloomFilterIndex {
794        source: index::bloom_filter::error::Error,
795        #[snafu(implicit)]
796        location: Location,
797    },
798
799    #[snafu(display("Failed to push index value"))]
800    PushIndexValue {
801        source: index::inverted_index::error::Error,
802        #[snafu(implicit)]
803        location: Location,
804    },
805
806    #[snafu(display("Failed to write index completely"))]
807    IndexFinish {
808        source: index::inverted_index::error::Error,
809        #[snafu(implicit)]
810        location: Location,
811    },
812
813    #[snafu(display("Operate on aborted index"))]
814    OperateAbortedIndex {
815        #[snafu(implicit)]
816        location: Location,
817    },
818
819    #[snafu(display("Failed to read puffin blob"))]
820    PuffinReadBlob {
821        source: puffin::error::Error,
822        #[snafu(implicit)]
823        location: Location,
824    },
825
826    #[snafu(display("Failed to add blob to puffin file"))]
827    PuffinAddBlob {
828        source: puffin::error::Error,
829        #[snafu(implicit)]
830        location: Location,
831    },
832
833    #[snafu(display("Failed to clean dir {dir}"))]
834    CleanDir {
835        dir: String,
836        #[snafu(source)]
837        error: std::io::Error,
838        #[snafu(implicit)]
839        location: Location,
840    },
841
842    #[snafu(display("Invalid config, {reason}"))]
843    InvalidConfig {
844        reason: String,
845        #[snafu(implicit)]
846        location: Location,
847    },
848
849    #[snafu(display(
850        "Stale log entry found during replay, region: {}, flushed: {}, replayed: {}",
851        region_id,
852        flushed_entry_id,
853        unexpected_entry_id
854    ))]
855    StaleLogEntry {
856        region_id: RegionId,
857        flushed_entry_id: u64,
858        unexpected_entry_id: u64,
859    },
860
861    #[snafu(display(
862        "Failed to download file, region_id: {}, file_id: {}, file_type: {:?}",
863        region_id,
864        file_id,
865        file_type,
866    ))]
867    Download {
868        region_id: RegionId,
869        file_id: FileId,
870        file_type: FileType,
871        #[snafu(source)]
872        error: std::io::Error,
873        #[snafu(implicit)]
874        location: Location,
875    },
876
877    #[snafu(display(
878        "Failed to upload file, region_id: {}, file_id: {}, file_type: {:?}",
879        region_id,
880        file_id,
881        file_type,
882    ))]
883    Upload {
884        region_id: RegionId,
885        file_id: FileId,
886        file_type: FileType,
887        #[snafu(source)]
888        error: std::io::Error,
889        #[snafu(implicit)]
890        location: Location,
891    },
892
893    #[snafu(display("Failed to create directory {}", dir))]
894    CreateDir {
895        dir: String,
896        #[snafu(source)]
897        error: std::io::Error,
898    },
899
900    #[snafu(display("Record batch error"))]
901    RecordBatch {
902        source: common_recordbatch::error::Error,
903        #[snafu(implicit)]
904        location: Location,
905    },
906
907    #[snafu(display("BiErrors, first: {first}, second: {second}"))]
908    BiErrors {
909        first: Box<Error>,
910        second: Box<Error>,
911        #[snafu(implicit)]
912        location: Location,
913    },
914
915    #[snafu(display("Encode null value"))]
916    IndexEncodeNull {
917        #[snafu(implicit)]
918        location: Location,
919    },
920
921    #[snafu(display("Failed to encode memtable to Parquet bytes"))]
922    EncodeMemtable {
923        #[snafu(source)]
924        error: parquet::errors::ParquetError,
925        #[snafu(implicit)]
926        location: Location,
927    },
928
929    #[snafu(display("Partition {} out of range, {} in total", given, all))]
930    PartitionOutOfRange {
931        given: usize,
932        all: usize,
933        #[snafu(implicit)]
934        location: Location,
935    },
936
937    #[snafu(display("Failed to iter data part"))]
938    ReadDataPart {
939        #[snafu(implicit)]
940        location: Location,
941        #[snafu(source)]
942        error: parquet::errors::ParquetError,
943    },
944
945    #[snafu(display("Failed to read row group in memtable"))]
946    DecodeArrowRowGroup {
947        #[snafu(source)]
948        error: ArrowError,
949        #[snafu(implicit)]
950        location: Location,
951    },
952
953    #[snafu(display("Invalid region options, {}", reason))]
954    InvalidRegionOptions {
955        reason: String,
956        #[snafu(implicit)]
957        location: Location,
958    },
959
960    #[snafu(display("checksum mismatch (actual: {}, expected: {})", actual, expected))]
961    ChecksumMismatch { actual: u32, expected: u32 },
962
963    #[snafu(display(
964        "No checkpoint found, region: {}, last_version: {}",
965        region_id,
966        last_version
967    ))]
968    NoCheckpoint {
969        region_id: RegionId,
970        last_version: ManifestVersion,
971        #[snafu(implicit)]
972        location: Location,
973    },
974
975    #[snafu(display(
976        "No manifests found in range: [{}..{}), region: {}, last_version: {}",
977        start_version,
978        end_version,
979        region_id,
980        last_version
981    ))]
982    NoManifests {
983        region_id: RegionId,
984        start_version: ManifestVersion,
985        end_version: ManifestVersion,
986        last_version: ManifestVersion,
987        #[snafu(implicit)]
988        location: Location,
989    },
990
991    #[snafu(display(
992        "Failed to install manifest to {}, region: {}, available manifest version: {}, last version: {}",
993        target_version,
994        region_id,
995        available_version,
996        last_version
997    ))]
998    InstallManifestTo {
999        region_id: RegionId,
1000        target_version: ManifestVersion,
1001        available_version: ManifestVersion,
1002        #[snafu(implicit)]
1003        location: Location,
1004        last_version: ManifestVersion,
1005    },
1006
1007    #[snafu(display("Region {} is stopped", region_id))]
1008    RegionStopped {
1009        region_id: RegionId,
1010        #[snafu(implicit)]
1011        location: Location,
1012    },
1013
1014    #[snafu(display(
1015        "Time range predicate overflows, timestamp: {:?}, target unit: {}",
1016        timestamp,
1017        unit
1018    ))]
1019    TimeRangePredicateOverflow {
1020        timestamp: Timestamp,
1021        unit: TimeUnit,
1022        #[snafu(implicit)]
1023        location: Location,
1024    },
1025
1026    #[snafu(display("Failed to open region"))]
1027    OpenRegion {
1028        #[snafu(implicit)]
1029        location: Location,
1030        source: Arc<Error>,
1031    },
1032
1033    #[snafu(display(
1034        "Region {} does not satisfy requirement '{}': {}",
1035        region_id,
1036        requirement,
1037        reason
1038    ))]
1039    RegionRequirement {
1040        region_id: RegionId,
1041        requirement: &'static str,
1042        reason: &'static str,
1043        #[snafu(implicit)]
1044        location: Location,
1045    },
1046
1047    #[snafu(display("Failed to parse job id"))]
1048    ParseJobId {
1049        #[snafu(implicit)]
1050        location: Location,
1051        #[snafu(source)]
1052        error: uuid::Error,
1053    },
1054
1055    #[snafu(display("Operation is not supported: {}", err_msg))]
1056    UnsupportedOperation {
1057        err_msg: String,
1058        #[snafu(implicit)]
1059        location: Location,
1060    },
1061
1062    #[snafu(display(
1063        "Failed to remotely compact region {} by job {:?} due to {}",
1064        region_id,
1065        job_id,
1066        reason
1067    ))]
1068    RemoteCompaction {
1069        region_id: RegionId,
1070        job_id: Option<JobId>,
1071        reason: String,
1072        #[snafu(implicit)]
1073        location: Location,
1074    },
1075
1076    #[snafu(display("Failed to initialize puffin stager"))]
1077    PuffinInitStager {
1078        source: puffin::error::Error,
1079        #[snafu(implicit)]
1080        location: Location,
1081    },
1082
1083    #[snafu(display("Failed to purge puffin stager"))]
1084    PuffinPurgeStager {
1085        source: puffin::error::Error,
1086        #[snafu(implicit)]
1087        location: Location,
1088    },
1089
1090    #[snafu(display("Failed to build puffin reader"))]
1091    PuffinBuildReader {
1092        source: puffin::error::Error,
1093        #[snafu(implicit)]
1094        location: Location,
1095    },
1096
1097    #[snafu(display("Failed to retrieve index options from column metadata"))]
1098    IndexOptions {
1099        #[snafu(implicit)]
1100        location: Location,
1101        source: datatypes::error::Error,
1102        column_name: String,
1103    },
1104
1105    #[snafu(display("Failed to retrieve compression options of column {column_name}"))]
1106    CompressionOptions {
1107        #[snafu(implicit)]
1108        location: Location,
1109        source: datatypes::error::Error,
1110        column_name: String,
1111    },
1112
1113    #[snafu(display("Failed to create fulltext index creator"))]
1114    CreateFulltextCreator {
1115        source: index::fulltext_index::error::Error,
1116        #[snafu(implicit)]
1117        location: Location,
1118    },
1119
1120    #[snafu(display("Failed to cast vector of {from} to {to}"))]
1121    CastVector {
1122        #[snafu(implicit)]
1123        location: Location,
1124        from: ConcreteDataType,
1125        to: ConcreteDataType,
1126        source: datatypes::error::Error,
1127    },
1128
1129    #[snafu(display("Failed to push text to fulltext index"))]
1130    FulltextPushText {
1131        source: index::fulltext_index::error::Error,
1132        #[snafu(implicit)]
1133        location: Location,
1134    },
1135
1136    #[snafu(display("Failed to finalize fulltext index creator"))]
1137    FulltextFinish {
1138        source: index::fulltext_index::error::Error,
1139        #[snafu(implicit)]
1140        location: Location,
1141    },
1142
1143    #[snafu(display("Failed to apply fulltext index"))]
1144    ApplyFulltextIndex {
1145        source: index::fulltext_index::error::Error,
1146        #[snafu(implicit)]
1147        location: Location,
1148    },
1149
1150    #[snafu(display("SST file {} does not contain valid stats info", file_path))]
1151    StatsNotPresent {
1152        file_path: String,
1153        #[snafu(implicit)]
1154        location: Location,
1155    },
1156
1157    #[snafu(display("Failed to decode stats of file {}", file_path))]
1158    DecodeStats {
1159        file_path: String,
1160        #[snafu(implicit)]
1161        location: Location,
1162    },
1163
1164    #[snafu(display("Invalid SST primary key range: {reason}"))]
1165    InvalidPrimaryKeyRange {
1166        reason: String,
1167        #[snafu(implicit)]
1168        location: Location,
1169    },
1170
1171    #[snafu(display("Failed to decode SST primary key range {endpoint} endpoint"))]
1172    DecodePrimaryKeyRange {
1173        endpoint: &'static str,
1174        source: mito_codec::error::Error,
1175        #[snafu(implicit)]
1176        location: Location,
1177    },
1178
1179    #[snafu(display("Region {} is busy", region_id))]
1180    RegionBusy {
1181        region_id: RegionId,
1182        #[snafu(implicit)]
1183        location: Location,
1184    },
1185
1186    #[snafu(display("Failed to get schema metadata"))]
1187    GetSchemaMetadata {
1188        source: common_meta::error::Error,
1189        #[snafu(implicit)]
1190        location: Location,
1191    },
1192
1193    #[snafu(display("Timeout"))]
1194    Timeout {
1195        #[snafu(source)]
1196        error: Elapsed,
1197        #[snafu(implicit)]
1198        location: Location,
1199    },
1200
1201    #[snafu(display("Failed to read file metadata"))]
1202    Metadata {
1203        #[snafu(source)]
1204        error: std::io::Error,
1205        #[snafu(implicit)]
1206        location: Location,
1207    },
1208
1209    #[snafu(display("Failed to push value to bloom filter"))]
1210    PushBloomFilterValue {
1211        source: index::bloom_filter::error::Error,
1212        #[snafu(implicit)]
1213        location: Location,
1214    },
1215
1216    #[snafu(display("Failed to finish bloom filter"))]
1217    BloomFilterFinish {
1218        source: index::bloom_filter::error::Error,
1219        #[snafu(implicit)]
1220        location: Location,
1221    },
1222
1223    #[snafu(display("Manual compaction is override by following operations."))]
1224    ManualCompactionOverride {},
1225
1226    #[snafu(display("Manual compaction is already running for region {region_id}."))]
1227    ManualCompactionAlreadyRunning { region_id: RegionId },
1228
1229    #[snafu(display("Compaction is cancelled."))]
1230    CompactionCancelled {},
1231
1232    #[snafu(display("Flush is cancelled."))]
1233    FlushCancelled {},
1234
1235    #[snafu(display("Compaction memory exhausted for region {region_id} (policy: {policy})",))]
1236    CompactionMemoryExhausted {
1237        region_id: RegionId,
1238        policy: String,
1239        #[snafu(source)]
1240        source: common_memory_manager::Error,
1241        #[snafu(implicit)]
1242        location: Location,
1243    },
1244
1245    #[snafu(display(
1246        "Incompatible WAL provider change. This is typically caused by changing WAL provider in database config file without completely cleaning existing files. Global provider: {}, region provider: {}",
1247        global,
1248        region
1249    ))]
1250    IncompatibleWalProviderChange { global: String, region: String },
1251
1252    #[snafu(display("Expected mito manifest info"))]
1253    MitoManifestInfo {
1254        #[snafu(implicit)]
1255        location: Location,
1256    },
1257
1258    #[snafu(display("Failed to scan series"))]
1259    ScanSeries {
1260        #[snafu(implicit)]
1261        location: Location,
1262        source: Arc<Error>,
1263    },
1264
1265    #[snafu(display("Partition {} scan multiple times", partition))]
1266    ScanMultiTimes {
1267        partition: usize,
1268        #[snafu(implicit)]
1269        location: Location,
1270    },
1271
1272    #[snafu(display("Invalid partition expression: {}", expr))]
1273    InvalidPartitionExpr {
1274        expr: String,
1275        #[snafu(implicit)]
1276        location: Location,
1277        source: partition::error::Error,
1278    },
1279
1280    #[snafu(display("Failed to decode bulk wal entry"))]
1281    ConvertBulkWalEntry {
1282        #[snafu(implicit)]
1283        location: Location,
1284        source: common_grpc::Error,
1285    },
1286
1287    #[snafu(display("Failed to encode"))]
1288    Encode {
1289        #[snafu(implicit)]
1290        location: Location,
1291        source: mito_codec::error::Error,
1292    },
1293
1294    #[snafu(display("Failed to decode"))]
1295    Decode {
1296        #[snafu(implicit)]
1297        location: Location,
1298        source: mito_codec::error::Error,
1299    },
1300
1301    #[snafu(display("Unexpected: {reason}"))]
1302    Unexpected {
1303        reason: String,
1304        #[snafu(implicit)]
1305        location: Location,
1306    },
1307
1308    #[cfg(feature = "enterprise")]
1309    #[snafu(display("Failed to scan external range"))]
1310    ScanExternalRange {
1311        source: BoxedError,
1312        #[snafu(implicit)]
1313        location: Location,
1314    },
1315
1316    #[snafu(display(
1317        "Inconsistent timestamp column length, expect: {}, actual: {}",
1318        expected,
1319        actual
1320    ))]
1321    InconsistentTimestampLength {
1322        expected: usize,
1323        actual: usize,
1324        #[snafu(implicit)]
1325        location: Location,
1326    },
1327
1328    #[snafu(display(
1329        "Too many files to read concurrently: {}, max allowed: {}",
1330        actual,
1331        max
1332    ))]
1333    TooManyFilesToRead {
1334        actual: usize,
1335        max: usize,
1336        #[snafu(implicit)]
1337        location: Location,
1338    },
1339
1340    #[snafu(display("Duration out of range: {input:?}"))]
1341    DurationOutOfRange {
1342        input: std::time::Duration,
1343        #[snafu(source)]
1344        error: chrono::OutOfRangeError,
1345        #[snafu(implicit)]
1346        location: Location,
1347    },
1348
1349    #[snafu(display("GC job permit exhausted"))]
1350    TooManyGcJobs {
1351        #[snafu(implicit)]
1352        location: Location,
1353    },
1354
1355    #[snafu(display(
1356        "Staging partition expr mismatch, manifest: {:?}, request: {}",
1357        manifest_expr,
1358        request_expr
1359    ))]
1360    StagingPartitionExprMismatch {
1361        manifest_expr: Option<String>,
1362        request_expr: String,
1363        #[snafu(implicit)]
1364        location: Location,
1365    },
1366
1367    #[snafu(display(
1368        "Invalid source and target region, source: {}, target: {}",
1369        source_region_id,
1370        target_region_id
1371    ))]
1372    InvalidSourceAndTargetRegion {
1373        source_region_id: RegionId,
1374        target_region_id: RegionId,
1375        #[snafu(implicit)]
1376        location: Location,
1377    },
1378
1379    #[snafu(display("Failed to prune file"))]
1380    PruneFile {
1381        source: Arc<Error>,
1382        #[snafu(implicit)]
1383        location: Location,
1384    },
1385
1386    #[snafu(display("Failed to cast column"))]
1387    CastColumn {
1388        #[snafu(source)]
1389        error: datafusion::error::DataFusionError,
1390        #[snafu(implicit)]
1391        location: Location,
1392    },
1393
1394    #[snafu(display("Failed to generate Arrow schema from Parquet file: {}", file))]
1395    ParquetToArrowSchema {
1396        file: String,
1397        #[snafu(source)]
1398        error: parquet::errors::ParquetError,
1399        #[snafu(implicit)]
1400        location: Location,
1401    },
1402
1403    #[snafu(display(
1404        "Region {} is in {:?} state, expect: Writable, Staging or Downgrading",
1405        region_id,
1406        state
1407    ))]
1408    FlushableRegionState {
1409        region_id: RegionId,
1410        state: RegionRoleState,
1411        #[snafu(implicit)]
1412        location: Location,
1413    },
1414}
1415
1416pub type Result<T, E = Error> = std::result::Result<T, E>;
1417
1418impl Error {
1419    /// Returns true if we need to fill default value for a region.
1420    pub(crate) fn is_fill_default(&self) -> bool {
1421        matches!(self, Error::FillDefault { .. })
1422    }
1423
1424    /// Returns true if the file is not found on the object store.
1425    pub(crate) fn is_object_not_found(&self) -> bool {
1426        match self {
1427            Error::OpenDal { error, .. } => error.kind() == ErrorKind::NotFound,
1428            Error::ManifestDeltaNotFound { .. } => true,
1429            _ => false,
1430        }
1431    }
1432
1433    /// Returns whether a failed manifest update may have been persisted.
1434    ///
1435    /// Unknown errors are treated conservatively because deleting output SSTs after an ambiguous
1436    /// manifest write could leave the manifest referencing missing files.
1437    pub(crate) fn may_have_persisted_manifest_update(&self) -> bool {
1438        match self {
1439            Error::UpdateManifest { .. }
1440            | Error::RegionState { .. }
1441            | Error::RegionTruncated { .. }
1442            | Error::RegionStopped { .. }
1443            | Error::SerdeJson { .. }
1444            | Error::CompressObject { .. } => false,
1445            Error::OpenDal { error, .. } => !matches!(
1446                error.kind(),
1447                ErrorKind::Unsupported
1448                    | ErrorKind::ConfigInvalid
1449                    | ErrorKind::NotFound
1450                    | ErrorKind::PermissionDenied
1451                    | ErrorKind::IsADirectory
1452                    | ErrorKind::NotADirectory
1453                    | ErrorKind::AlreadyExists
1454                    | ErrorKind::RateLimited
1455                    | ErrorKind::IsSameFile
1456                    | ErrorKind::ConditionNotMatch
1457                    | ErrorKind::RangeNotSatisfied
1458            ),
1459            _ => true,
1460        }
1461    }
1462}
1463
1464impl ErrorExt for Error {
1465    fn status_code(&self) -> StatusCode {
1466        use Error::*;
1467
1468        match self {
1469            DataTypeMismatch { source, .. } => source.status_code(),
1470            OpenDal { .. } | ManifestDeltaNotFound { .. } | ReadParquet { .. } => {
1471                StatusCode::StorageUnavailable
1472            }
1473            WriteWal { source, .. }
1474            | ReadWal { source, .. }
1475            | DeleteWal { source, .. }
1476            | WaitWalDurable { source, .. }
1477            | CheckWalHistory { source, .. } => source.status_code(),
1478            CompressObject { .. }
1479            | DecompressObject { .. }
1480            | SerdeJson { .. }
1481            | Utf8 { .. }
1482            | NewRecordBatch { .. }
1483            | RegionCorrupted { .. }
1484            | InconsistentFile { .. }
1485            | CreateDefault { .. }
1486            | InvalidParquet { .. }
1487            | OperateAbortedIndex { .. }
1488            | IndexEncodeNull { .. }
1489            | NoCheckpoint { .. }
1490            | NoManifests { .. }
1491            | FilesLost { .. }
1492            | InstallManifestTo { .. }
1493            | Unexpected { .. }
1494            | SerializeColumnMetadata { .. }
1495            | SerializeManifest { .. }
1496            | StagingPartitionExprMismatch { .. } => StatusCode::Unexpected,
1497
1498            RegionNotFound { .. } => StatusCode::RegionNotFound,
1499            ObjectStoreNotFound { .. }
1500            | InvalidScanIndex { .. }
1501            | InvalidMeta { .. }
1502            | InvalidRequest { .. }
1503            | PartitionExprVersionMismatch { .. }
1504            | FillDefault { .. }
1505            | ConvertColumnDataType { .. }
1506            | ColumnNotFound { .. }
1507            | InvalidMetadata { .. }
1508            | InvalidRegionOptions { .. }
1509            | InvalidWalReadRequest { .. }
1510            | PartitionOutOfRange { .. }
1511            | ParseJobId { .. }
1512            | DurationOutOfRange { .. }
1513            | MissingOldManifest { .. }
1514            | MissingNewManifest { .. }
1515            | MissingManifest { .. }
1516            | NoOldManifests { .. }
1517            | MissingPartitionExpr { .. }
1518            | SerializePartitionExpr { .. }
1519            | InvalidSourceAndTargetRegion { .. } => StatusCode::InvalidArguments,
1520
1521            IncrementalQueryStale { .. } | SnapshotFenceStale { .. } => StatusCode::RequestOutdated,
1522
1523            SequenceRangeUnsupported { .. } => StatusCode::Unsupported,
1524
1525            RegionSequenceDomainBroken { .. }
1526            | RegionMetadataNotFound { .. }
1527            | Join { .. }
1528            | WorkerStopped { .. }
1529            | Recv { .. }
1530            | DecodeWal { .. }
1531            | ComputeArrow { .. }
1532            | EvalPartitionFilter { .. }
1533            | MergeCandidateSeries { .. }
1534            | BiErrors { .. }
1535            | StopScheduler { .. }
1536            | ComputeVector { .. }
1537            | EncodeMemtable { .. }
1538            | CreateDir { .. }
1539            | ReadDataPart { .. }
1540            | BuildEntry { .. }
1541            | Metadata { .. }
1542            | CastColumn { .. }
1543            | MitoManifestInfo { .. }
1544            | ParquetToArrowSchema { .. } => StatusCode::Internal,
1545
1546            FetchManifests { source, .. } => source.status_code(),
1547
1548            OpenRegion { source, .. } => source.status_code(),
1549
1550            WriteParquet { .. } => StatusCode::StorageUnavailable,
1551            WriteGroup { source, .. } => source.status_code(),
1552            InvalidBatch { .. } => StatusCode::InvalidArguments,
1553            InvalidRecordBatch { .. } => StatusCode::InvalidArguments,
1554            ConvertVector { source, .. } => source.status_code(),
1555
1556            PrimaryKeyLengthMismatch { .. } => StatusCode::InvalidArguments,
1557            InvalidSender { .. } => StatusCode::InvalidArguments,
1558            InvalidSchedulerState { .. } => StatusCode::InvalidArguments,
1559            RegionRequirement { .. } => StatusCode::InvalidArguments,
1560            DeleteSsts { .. } | DeleteIndex { .. } | DeleteIndexes { .. } => {
1561                StatusCode::StorageUnavailable
1562            }
1563            FlushRegion { source, .. } | BuildIndexAsync { source, .. } => source.status_code(),
1564            RegionDropped { .. } => StatusCode::Cancelled,
1565            RegionClosed { .. } => StatusCode::Cancelled,
1566            StaleCompactionExecution { .. } => StatusCode::Cancelled,
1567            RegionTruncated { .. } => StatusCode::Cancelled,
1568            RejectWrite { .. } | RejectWriteInStaging { .. } => StatusCode::StorageUnavailable,
1569            CompactRegion { source, .. } => source.status_code(),
1570            EditRegion { source, .. } => source.status_code(),
1571            CompatReader { .. } => StatusCode::Unexpected,
1572            InvalidRegionRequest { source, .. } => source.status_code(),
1573            RegionState { .. } | UpdateManifest { .. } => StatusCode::RegionNotReady,
1574            JsonOptions { .. } => StatusCode::InvalidArguments,
1575            EmptyRegionDir { .. } | EmptyManifestDir { .. } => StatusCode::RegionNotFound,
1576            ConvertValue { source, .. } => source.status_code(),
1577            ApplyBloomFilterIndex { source, .. } => source.status_code(),
1578            InvalidPartitionExpr { source, .. } => source.status_code(),
1579            BuildIndexApplier { source, .. }
1580            | PushIndexValue { source, .. }
1581            | ApplyInvertedIndex { source, .. }
1582            | IndexFinish { source, .. } => source.status_code(),
1583            PuffinReadBlob { source, .. }
1584            | PuffinAddBlob { source, .. }
1585            | PuffinInitStager { source, .. }
1586            | PuffinBuildReader { source, .. }
1587            | PuffinPurgeStager { source, .. } => source.status_code(),
1588            CleanDir { .. } => StatusCode::Unexpected,
1589            InvalidConfig { .. } => StatusCode::InvalidArguments,
1590            StaleLogEntry { .. }
1591            | InvalidNativeHistogramSubfield { .. }
1592            | InvalidNativeHistogramFieldId { .. } => StatusCode::Unexpected,
1593
1594            External { source, .. } => source.status_code(),
1595
1596            RecordBatch { source, .. } => source.status_code(),
1597
1598            Download { .. } | Upload { .. } => StatusCode::StorageUnavailable,
1599            ChecksumMismatch { .. } => StatusCode::Unexpected,
1600            RegionStopped { .. } => StatusCode::RegionNotReady,
1601            TimeRangePredicateOverflow { .. } => StatusCode::InvalidArguments,
1602            UnsupportedOperation { .. } => StatusCode::Unsupported,
1603            RemoteCompaction { .. } => StatusCode::Unexpected,
1604
1605            IndexOptions { source, .. } => source.status_code(),
1606            CompressionOptions { source, .. } => source.status_code(),
1607            CreateFulltextCreator { source, .. } => source.status_code(),
1608            CastVector { source, .. } => source.status_code(),
1609            FulltextPushText { source, .. }
1610            | FulltextFinish { source, .. }
1611            | ApplyFulltextIndex { source, .. } => source.status_code(),
1612            DecodeStats { .. }
1613            | StatsNotPresent { .. }
1614            | InvalidPrimaryKeyRange { .. }
1615            | DecodePrimaryKeyRange { .. } => StatusCode::Internal,
1616            RegionBusy { .. } => StatusCode::RegionBusy,
1617            GetSchemaMetadata { source, .. } => source.status_code(),
1618            Timeout { .. } => StatusCode::Cancelled,
1619
1620            DecodeArrowRowGroup { .. } => StatusCode::Internal,
1621
1622            PushBloomFilterValue { source, .. } | BloomFilterFinish { source, .. } => {
1623                source.status_code()
1624            }
1625
1626            ManualCompactionOverride {} | CompactionCancelled {} | FlushCancelled {} => {
1627                StatusCode::Cancelled
1628            }
1629
1630            // A concurrent manual compaction fails fast instead of being queued;
1631            // the conflict is reported to the caller, which decides whether to
1632            // issue a new request after the running one finishes.
1633            ManualCompactionAlreadyRunning { .. } => StatusCode::RegionBusy,
1634
1635            CompactionMemoryExhausted { source, .. } => source.status_code(),
1636
1637            IncompatibleWalProviderChange { .. } => StatusCode::InvalidArguments,
1638
1639            ScanSeries { source, .. } => source.status_code(),
1640
1641            ScanMultiTimes { .. } => StatusCode::InvalidArguments,
1642            ConvertBulkWalEntry { source, .. } => source.status_code(),
1643
1644            Encode { source, .. } | Decode { source, .. } => source.status_code(),
1645
1646            #[cfg(feature = "enterprise")]
1647            ScanExternalRange { source, .. } => source.status_code(),
1648
1649            InconsistentTimestampLength { .. } => StatusCode::InvalidArguments,
1650
1651            TooManyFilesToRead { .. } | TooManyGcJobs { .. } => StatusCode::RateLimited,
1652
1653            PruneFile { source, .. } => source.status_code(),
1654
1655            FlushableRegionState { .. } => StatusCode::RegionNotReady,
1656        }
1657    }
1658
1659    fn as_any(&self) -> &dyn Any {
1660        self
1661    }
1662
1663    fn retry_hint(&self) -> RetryHint {
1664        use Error::*;
1665
1666        match self {
1667            PartitionExprVersionMismatch { .. }
1668            | RegionNotFound { .. }
1669            | RejectWriteInStaging { .. }
1670            | RegionState {
1671                state: RegionRoleState::Follower,
1672                ..
1673            } => RetryHint::RetryAfterRouteRefresh,
1674            ReadParquet { .. }
1675            | WriteParquet { .. }
1676            | RejectWrite { .. }
1677            | Download { .. }
1678            | Upload { .. }
1679            | RegionState { .. }
1680            | UpdateManifest { .. }
1681            | RegionStopped { .. }
1682            | RegionBusy { .. }
1683            | ManualCompactionAlreadyRunning { .. }
1684            | FlushableRegionState { .. }
1685            | ManifestDeltaNotFound { .. } => RetryHint::Retryable,
1686
1687            OpenDal { error, .. }
1688            | DeleteSsts { error, .. }
1689            | DeleteIndex { error, .. }
1690            | DeleteIndexes { error, .. } => retry_hint_from_opendal_error(error),
1691
1692            WriteWal { source, .. }
1693            | ReadWal { source, .. }
1694            | DeleteWal { source, .. }
1695            | WaitWalDurable { source, .. }
1696            | CheckWalHistory { source, .. }
1697            | FetchManifests { source, .. }
1698            | External { source, .. } => source.retry_hint(),
1699
1700            OpenRegion { source, .. }
1701            | WriteGroup { source, .. }
1702            | FlushRegion { source, .. }
1703            | BuildIndexAsync { source, .. }
1704            | CompactRegion { source, .. }
1705            | EditRegion { source, .. }
1706            | ScanSeries { source, .. }
1707            | PruneFile { source, .. } => source.retry_hint(),
1708
1709            DataTypeMismatch { source, .. }
1710            | ConvertVector { source, .. }
1711            | ConvertValue { source, .. }
1712            | IndexOptions { source, .. }
1713            | CompressionOptions { source, .. }
1714            | CastVector { source, .. } => source.retry_hint(),
1715
1716            BuildIndexApplier { source, .. }
1717            | PushIndexValue { source, .. }
1718            | ApplyInvertedIndex { source, .. }
1719            | IndexFinish { source, .. } => source.retry_hint(),
1720
1721            ApplyBloomFilterIndex { source, .. }
1722            | PushBloomFilterValue { source, .. }
1723            | BloomFilterFinish { source, .. } => source.retry_hint(),
1724
1725            PuffinReadBlob { source, .. }
1726            | PuffinAddBlob { source, .. }
1727            | PuffinInitStager { source, .. }
1728            | PuffinBuildReader { source, .. }
1729            | PuffinPurgeStager { source, .. } => source.retry_hint(),
1730
1731            CreateFulltextCreator { source, .. }
1732            | FulltextPushText { source, .. }
1733            | FulltextFinish { source, .. }
1734            | ApplyFulltextIndex { source, .. } => source.retry_hint(),
1735
1736            InvalidRegionRequest { source, .. } => source.retry_hint(),
1737            InvalidPartitionExpr { source, .. } => source.retry_hint(),
1738            RecordBatch { source, .. } => source.retry_hint(),
1739            GetSchemaMetadata { source, .. } => source.retry_hint(),
1740            CompactionMemoryExhausted { source, .. } => source.retry_hint(),
1741            ConvertBulkWalEntry { source, .. } => source.retry_hint(),
1742            Encode { source, .. } | Decode { source, .. } => source.retry_hint(),
1743
1744            #[cfg(feature = "enterprise")]
1745            ScanExternalRange { source, .. } => source.retry_hint(),
1746
1747            _ => RetryHint::NonRetryable,
1748        }
1749    }
1750}
1751
1752#[cfg(test)]
1753mod tests {
1754    use snafu::IntoError;
1755
1756    use super::*;
1757    use crate::region::RegionLeaderState;
1758
1759    #[test]
1760    fn test_partition_version_mismatch_retry_hint() {
1761        let error = PartitionExprVersionMismatchSnafu {
1762            region_id: RegionId::new(1, 1),
1763            request_version: 1u64,
1764            expected_version: 2u64,
1765        }
1766        .build();
1767        assert_eq!(error.status_code(), StatusCode::InvalidArguments);
1768        assert_eq!(error.retry_hint(), RetryHint::RetryAfterRouteRefresh);
1769    }
1770
1771    #[test]
1772    fn test_region_state_retry_hint() {
1773        let region_id = RegionId::new(1, 1);
1774        let writable = RegionRoleState::Leader(RegionLeaderState::Writable);
1775        let follower_error = RegionStateSnafu {
1776            region_id,
1777            state: RegionRoleState::Follower,
1778            expect: writable,
1779        }
1780        .build();
1781        assert_eq!(
1782            follower_error.retry_hint(),
1783            RetryHint::RetryAfterRouteRefresh
1784        );
1785        assert_eq!(follower_error.status_code(), StatusCode::RegionNotReady);
1786        let leader_error = RegionStateSnafu {
1787            region_id,
1788            state: writable,
1789            expect: RegionRoleState::Follower,
1790        }
1791        .build();
1792        assert_eq!(leader_error.retry_hint(), RetryHint::Retryable);
1793        for (error, status, hint) in [
1794            (
1795                RejectWriteSnafu { region_id }.build(),
1796                StatusCode::StorageUnavailable,
1797                RetryHint::Retryable,
1798            ),
1799            (
1800                RejectWriteInStagingSnafu { region_id }.build(),
1801                StatusCode::StorageUnavailable,
1802                RetryHint::RetryAfterRouteRefresh,
1803            ),
1804            (
1805                RegionNotFoundSnafu { region_id }.build(),
1806                StatusCode::RegionNotFound,
1807                RetryHint::RetryAfterRouteRefresh,
1808            ),
1809        ] {
1810            assert_eq!(error.status_code(), status);
1811            assert_eq!(error.retry_hint(), hint);
1812        }
1813    }
1814
1815    #[test]
1816    fn test_manifest_update_persistence() {
1817        let rejected_kinds = [
1818            ErrorKind::Unsupported,
1819            ErrorKind::ConfigInvalid,
1820            ErrorKind::NotFound,
1821            ErrorKind::PermissionDenied,
1822            ErrorKind::IsADirectory,
1823            ErrorKind::NotADirectory,
1824            ErrorKind::AlreadyExists,
1825            ErrorKind::RateLimited,
1826            ErrorKind::IsSameFile,
1827            ErrorKind::ConditionNotMatch,
1828            ErrorKind::RangeNotSatisfied,
1829        ];
1830        for kind in rejected_kinds {
1831            let error = OpenDalSnafu {}.into_error(object_store::Error::new(kind, "test"));
1832            assert!(
1833                !error.may_have_persisted_manifest_update(),
1834                "error kind {kind:?} should prove the manifest was not persisted"
1835            );
1836        }
1837
1838        let error =
1839            OpenDalSnafu {}.into_error(object_store::Error::new(ErrorKind::Unexpected, "test"));
1840        assert!(error.may_have_persisted_manifest_update());
1841
1842        let region_id = RegionId::new(1, 1);
1843        let error = RegionTruncatedSnafu { region_id }.build();
1844        assert!(!error.may_have_persisted_manifest_update());
1845
1846        // This can be raised while applying an edit after its manifest file was saved.
1847        let error = RegionMetadataNotFoundSnafu {}.build();
1848        assert!(error.may_have_persisted_manifest_update());
1849    }
1850}
mito2/error.rs

mito2/
error.rs
