mito2/
error.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::any::Any;
16use std::sync::Arc;
17
18use common_datasource::compression::CompressionType;
19use common_error::ext::{BoxedError, ErrorExt, RetryHint};
20use common_error::status_code::StatusCode;
21use common_macro::stack_trace_debug;
22use common_memory_manager;
23use common_runtime::JoinError;
24use common_time::Timestamp;
25use common_time::timestamp::TimeUnit;
26use datatypes::arrow::error::ArrowError;
27use datatypes::prelude::ConcreteDataType;
28use object_store::ErrorKind;
29use object_store::error::retry_hint_from_opendal_error;
30use partition::error::Error as PartitionError;
31use prost::DecodeError;
32use snafu::{Location, Snafu};
33use store_api::ManifestVersion;
34use store_api::logstore::provider::Provider;
35use store_api::storage::{FileId, RegionId};
36use tokio::time::error::Elapsed;
37
38use crate::cache::file_cache::FileType;
39use crate::region::RegionRoleState;
40use crate::schedule::remote_job_scheduler::JobId;
41use crate::worker::WorkerId;
42
43#[derive(Snafu)]
44#[snafu(visibility(pub))]
45#[stack_trace_debug]
46pub enum Error {
47    #[snafu(display("Unexpected data type"))]
48    DataTypeMismatch {
49        source: datatypes::error::Error,
50        #[snafu(implicit)]
51        location: Location,
52    },
53
54    #[snafu(display("External error, context: {}", context))]
55    External {
56        source: BoxedError,
57        context: String,
58        #[snafu(implicit)]
59        location: Location,
60    },
61
62    #[snafu(display("OpenDAL operator failed"))]
63    OpenDal {
64        #[snafu(implicit)]
65        location: Location,
66        #[snafu(source)]
67        error: object_store::Error,
68    },
69
70    #[snafu(display(
71        "Manifest delta {} disappeared after it was listed, path: {}",
72        version,
73        path
74    ))]
75    ManifestDeltaNotFound {
76        version: ManifestVersion,
77        path: String,
78        #[snafu(source)]
79        error: object_store::Error,
80        #[snafu(implicit)]
81        location: Location,
82    },
83
84    #[snafu(display("Fail to compress object by {}, path: {}", compress_type, path))]
85    CompressObject {
86        compress_type: CompressionType,
87        path: String,
88        #[snafu(source)]
89        error: std::io::Error,
90    },
91
92    #[snafu(display("Fail to decompress object by {}, path: {}", compress_type, path))]
93    DecompressObject {
94        compress_type: CompressionType,
95        path: String,
96        #[snafu(source)]
97        error: std::io::Error,
98    },
99
100    #[snafu(display("Failed to ser/de json object"))]
101    SerdeJson {
102        #[snafu(implicit)]
103        location: Location,
104        #[snafu(source)]
105        error: serde_json::Error,
106    },
107
108    #[snafu(display("Failed to serialize column metadata"))]
109    SerializeColumnMetadata {
110        #[snafu(source)]
111        error: serde_json::Error,
112        #[snafu(implicit)]
113        location: Location,
114    },
115
116    #[snafu(display("Failed to serialize manifest, region_id: {}", region_id))]
117    SerializeManifest {
118        region_id: RegionId,
119        #[snafu(source)]
120        error: serde_json::Error,
121        #[snafu(implicit)]
122        location: Location,
123    },
124
125    #[snafu(display("Invalid scan index, start: {}, end: {}", start, end))]
126    InvalidScanIndex {
127        start: ManifestVersion,
128        end: ManifestVersion,
129        #[snafu(implicit)]
130        location: Location,
131    },
132
133    #[snafu(display("Invalid UTF-8 content"))]
134    Utf8 {
135        #[snafu(implicit)]
136        location: Location,
137        #[snafu(source)]
138        error: std::str::Utf8Error,
139    },
140
141    #[snafu(display("Cannot find RegionMetadata"))]
142    RegionMetadataNotFound {
143        #[snafu(implicit)]
144        location: Location,
145    },
146
147    #[snafu(display("Failed to join handle"))]
148    Join {
149        #[snafu(source)]
150        error: common_runtime::JoinError,
151        #[snafu(implicit)]
152        location: Location,
153    },
154
155    #[snafu(display("Worker {} is stopped", id))]
156    WorkerStopped {
157        id: WorkerId,
158        #[snafu(implicit)]
159        location: Location,
160    },
161
162    #[snafu(display("Failed to recv result"))]
163    Recv {
164        #[snafu(source)]
165        error: tokio::sync::oneshot::error::RecvError,
166        #[snafu(implicit)]
167        location: Location,
168    },
169
170    #[snafu(display("Invalid metadata, {}", reason))]
171    InvalidMeta {
172        reason: String,
173        #[snafu(implicit)]
174        location: Location,
175    },
176
177    #[snafu(display("Invalid region metadata"))]
178    InvalidMetadata {
179        source: store_api::metadata::MetadataError,
180        #[snafu(implicit)]
181        location: Location,
182    },
183
184    #[snafu(display("Failed to create RecordBatch from vectors"))]
185    NewRecordBatch {
186        #[snafu(implicit)]
187        location: Location,
188        #[snafu(source)]
189        error: ArrowError,
190    },
191
192    #[snafu(display("Failed to read parquet file, path: {}", path))]
193    ReadParquet {
194        path: String,
195        #[snafu(source)]
196        error: parquet::errors::ParquetError,
197        #[snafu(implicit)]
198        location: Location,
199    },
200
201    #[snafu(display("Failed to write parquet file"))]
202    WriteParquet {
203        #[snafu(source)]
204        error: parquet::errors::ParquetError,
205        #[snafu(implicit)]
206        location: Location,
207    },
208
209    #[snafu(display(
210        "Cannot assign a stable field id to native histogram sub-field '{}' of column id {} (unknown sub-field name or derived id overflows i32)",
211        field_name,
212        column_id
213    ))]
214    InvalidNativeHistogramSubfield {
215        column_id: i32,
216        field_name: String,
217        #[snafu(implicit)]
218        location: Location,
219    },
220
221    #[snafu(display(
222        "Native histogram column '{}' has no usable PARQUET:field_id to namespace its sub-field ids (missing, malformed, or exceeds i32::MAX)",
223        field_name
224    ))]
225    InvalidNativeHistogramFieldId {
226        field_name: String,
227        #[snafu(implicit)]
228        location: Location,
229    },
230
231    #[snafu(display("Region {} not found", region_id))]
232    RegionNotFound {
233        region_id: RegionId,
234        #[snafu(implicit)]
235        location: Location,
236    },
237
238    #[snafu(display("Object store not found: {}", object_store))]
239    ObjectStoreNotFound {
240        object_store: String,
241        #[snafu(implicit)]
242        location: Location,
243    },
244
245    #[snafu(display("Region {} is corrupted, reason: {}", region_id, reason))]
246    RegionCorrupted {
247        region_id: RegionId,
248        reason: String,
249        #[snafu(implicit)]
250        location: Location,
251    },
252
253    #[snafu(display("Invalid request to region {}, reason: {}", region_id, reason))]
254    InvalidRequest {
255        region_id: RegionId,
256        reason: String,
257        #[snafu(implicit)]
258        location: Location,
259    },
260
261    #[snafu(display(
262        "STALE_CURSOR: incremental query stale, region: {}, given_seq: {}, min_readable_seq: {}, retry_hint: FALLBACK_FULL_RECOMPUTE",
263        region_id,
264        given_seq,
265        min_readable_seq
266    ))]
267    IncrementalQueryStale {
268        region_id: RegionId,
269        given_seq: u64,
270        min_readable_seq: u64,
271        #[snafu(implicit)]
272        location: Location,
273    },
274
275    #[snafu(display(
276        "STALE_SNAPSHOT_FENCE: snapshot upper bound stale, region: {}, given_seq: {}, min_enforceable_seq: {}, retry_hint: REBIND_SNAPSHOT_FENCE",
277        region_id,
278        given_seq,
279        min_enforceable_seq
280    ))]
281    SnapshotFenceStale {
282        region_id: RegionId,
283        given_seq: u64,
284        min_enforceable_seq: u64,
285        #[snafu(implicit)]
286        location: Location,
287    },
288
289    #[snafu(display(
290        "SEQUENCE_RANGE_UNSUPPORTED: exact sequence-range read unsupported, region: {}, min_seq: {}, max_seq: {}, reason: {}, retry_hint: FALLBACK_MEMTABLE_ONLY_OR_FULL_RECOMPUTE",
291        region_id,
292        min_seq,
293        max_seq,
294        reason
295    ))]
296    SequenceRangeUnsupported {
297        region_id: RegionId,
298        min_seq: u64,
299        max_seq: u64,
300        reason: String,
301        #[snafu(implicit)]
302        location: Location,
303    },
304
305    #[snafu(display(
306        "region {} is unusable for sequence reads: file {} declares region {}",
307        region_id,
308        file_id,
309        file_region_id
310    ))]
311    RegionSequenceDomainBroken {
312        region_id: RegionId,
313        file_region_id: RegionId,
314        file_id: FileId,
315        #[snafu(implicit)]
316        location: Location,
317    },
318
319    #[snafu(display("Old manifest missing for region {}", region_id))]
320    MissingOldManifest {
321        region_id: RegionId,
322        #[snafu(implicit)]
323        location: Location,
324    },
325
326    #[snafu(display("New manifest missing for region {}", region_id))]
327    MissingNewManifest {
328        region_id: RegionId,
329        #[snafu(implicit)]
330        location: Location,
331    },
332
333    #[snafu(display("Manifest missing for region {}", region_id))]
334    MissingManifest {
335        region_id: RegionId,
336        #[snafu(implicit)]
337        location: Location,
338    },
339
340    #[snafu(display("File consistency check failed for file {}: {}", file_id, reason))]
341    InconsistentFile {
342        file_id: FileId,
343        reason: String,
344        #[snafu(implicit)]
345        location: Location,
346    },
347
348    #[snafu(display("Files lost during remapping: old={}, new={}", old_count, new_count))]
349    FilesLost {
350        old_count: usize,
351        new_count: usize,
352        #[snafu(implicit)]
353        location: Location,
354    },
355
356    #[snafu(display("No old manifests provided (need at least one for template)"))]
357    NoOldManifests {
358        #[snafu(implicit)]
359        location: Location,
360    },
361
362    #[snafu(display("Failed to fetch manifests"))]
363    FetchManifests {
364        #[snafu(implicit)]
365        location: Location,
366        source: BoxedError,
367    },
368
369    #[snafu(display("Partition expression missing for region {}", region_id))]
370    MissingPartitionExpr {
371        region_id: RegionId,
372        #[snafu(implicit)]
373        location: Location,
374    },
375
376    #[snafu(display("Failed to serialize partition expression: {}", source))]
377    SerializePartitionExpr {
378        #[snafu(source)]
379        source: PartitionError,
380        #[snafu(implicit)]
381        location: Location,
382    },
383
384    #[snafu(display(
385        "Failed to convert ConcreteDataType to ColumnDataType, reason: {}",
386        reason
387    ))]
388    ConvertColumnDataType {
389        reason: String,
390        source: api::error::Error,
391        #[snafu(implicit)]
392        location: Location,
393    },
394
395    /// An error type to indicate that schema is changed and we need
396    /// to fill default values again.
397    #[snafu(display("Need to fill default value for region {}", region_id))]
398    FillDefault {
399        region_id: RegionId,
400        // The error is for internal use so we don't need a location.
401    },
402
403    #[snafu(display(
404        "Failed to create default value for column {} of region {}",
405        column,
406        region_id
407    ))]
408    CreateDefault {
409        region_id: RegionId,
410        column: String,
411        source: datatypes::Error,
412        #[snafu(implicit)]
413        location: Location,
414    },
415
416    #[snafu(display("Failed to build entry, region_id: {}", region_id))]
417    BuildEntry {
418        region_id: RegionId,
419        #[snafu(implicit)]
420        location: Location,
421        source: BoxedError,
422    },
423
424    #[snafu(display("Failed to write WAL"))]
425    WriteWal {
426        #[snafu(implicit)]
427        location: Location,
428        source: BoxedError,
429    },
430
431    #[snafu(display("Failed to read WAL, provider: {}", provider))]
432    ReadWal {
433        provider: Provider,
434        #[snafu(implicit)]
435        location: Location,
436        source: BoxedError,
437    },
438
439    #[snafu(display("Failed to decode WAL entry, region_id: {}", region_id))]
440    DecodeWal {
441        region_id: RegionId,
442        #[snafu(implicit)]
443        location: Location,
444        #[snafu(source)]
445        error: DecodeError,
446    },
447
448    #[snafu(display("Failed to delete WAL, region_id: {}", region_id))]
449    DeleteWal {
450        region_id: RegionId,
451        #[snafu(implicit)]
452        location: Location,
453        source: BoxedError,
454    },
455
456    #[snafu(display("Failed to wait for the WAL to be durable, region_id: {}", region_id))]
457    WaitWalDurable {
458        region_id: RegionId,
459        #[snafu(implicit)]
460        location: Location,
461        source: BoxedError,
462    },
463
464    #[snafu(display("Failed to check the WAL history, region_id: {}", region_id))]
465    CheckWalHistory {
466        region_id: RegionId,
467        #[snafu(implicit)]
468        location: Location,
469        source: BoxedError,
470    },
471
472    // Shared error for each writer in the write group.
473    #[snafu(display("Failed to write region"))]
474    WriteGroup { source: Arc<Error> },
475
476    #[snafu(display("Invalid parquet SST file {}, reason: {}", file, reason))]
477    InvalidParquet {
478        file: String,
479        reason: String,
480        #[snafu(implicit)]
481        location: Location,
482    },
483
484    #[snafu(display("Invalid batch, {}", reason))]
485    InvalidBatch {
486        reason: String,
487        #[snafu(implicit)]
488        location: Location,
489    },
490
491    #[snafu(display("Invalid arrow record batch, {}", reason))]
492    InvalidRecordBatch {
493        reason: String,
494        #[snafu(implicit)]
495        location: Location,
496    },
497
498    #[snafu(display("Invalid wal read request, {}", reason))]
499    InvalidWalReadRequest {
500        reason: String,
501        #[snafu(implicit)]
502        location: Location,
503    },
504
505    #[snafu(display("Failed to convert array to vector"))]
506    ConvertVector {
507        #[snafu(implicit)]
508        location: Location,
509        source: datatypes::error::Error,
510    },
511
512    #[snafu(display("Failed to compute arrow arrays"))]
513    ComputeArrow {
514        #[snafu(implicit)]
515        location: Location,
516        #[snafu(source)]
517        error: datatypes::arrow::error::ArrowError,
518    },
519
520    #[snafu(display("Failed to evaluate partition filter"))]
521    EvalPartitionFilter {
522        #[snafu(implicit)]
523        location: Location,
524        #[snafu(source)]
525        error: datafusion::error::DataFusionError,
526    },
527
528    #[snafu(display("Failed to merge candidate series"))]
529    MergeCandidateSeries {
530        #[snafu(implicit)]
531        location: Location,
532        #[snafu(source)]
533        error: datafusion::error::DataFusionError,
534    },
535
536    #[snafu(display("Failed to compute vector"))]
537    ComputeVector {
538        #[snafu(implicit)]
539        location: Location,
540        source: datatypes::error::Error,
541    },
542
543    #[snafu(display("Primary key length mismatch, expect: {}, actual: {}", expect, actual))]
544    PrimaryKeyLengthMismatch {
545        expect: usize,
546        actual: usize,
547        #[snafu(implicit)]
548        location: Location,
549    },
550
551    #[snafu(display("Invalid sender",))]
552    InvalidSender {
553        #[snafu(implicit)]
554        location: Location,
555    },
556
557    #[snafu(display("Invalid scheduler state"))]
558    InvalidSchedulerState {
559        #[snafu(implicit)]
560        location: Location,
561    },
562
563    #[snafu(display("Failed to stop scheduler"))]
564    StopScheduler {
565        #[snafu(source)]
566        error: JoinError,
567        #[snafu(implicit)]
568        location: Location,
569    },
570
571    #[snafu(display(
572        "Failed to batch delete SST files, region id: {}, file ids: {:?}",
573        region_id,
574        file_ids
575    ))]
576    DeleteSsts {
577        region_id: RegionId,
578        file_ids: Vec<FileId>,
579        #[snafu(source)]
580        error: object_store::Error,
581        #[snafu(implicit)]
582        location: Location,
583    },
584
585    #[snafu(display("Failed to delete index file, file id: {}", file_id))]
586    DeleteIndex {
587        file_id: FileId,
588        #[snafu(source)]
589        error: object_store::Error,
590        #[snafu(implicit)]
591        location: Location,
592    },
593
594    #[snafu(display("Failed to batch delete index files, file ids: {:?}", file_ids))]
595    DeleteIndexes {
596        file_ids: Vec<FileId>,
597        #[snafu(source)]
598        error: object_store::Error,
599        #[snafu(implicit)]
600        location: Location,
601    },
602
603    #[snafu(display("Failed to flush region {}", region_id))]
604    FlushRegion {
605        region_id: RegionId,
606        source: Arc<Error>,
607        #[snafu(implicit)]
608        location: Location,
609    },
610
611    #[snafu(display("Region {} is dropped", region_id))]
612    RegionDropped {
613        region_id: RegionId,
614        #[snafu(implicit)]
615        location: Location,
616    },
617
618    #[snafu(display("Region {} is closed", region_id))]
619    RegionClosed {
620        region_id: RegionId,
621        #[snafu(implicit)]
622        location: Location,
623    },
624
625    #[snafu(display(
626        "Stale compaction execution for region {}, the region may have been reopened, truncated or the compaction was superseded",
627        region_id
628    ))]
629    StaleCompactionExecution {
630        region_id: RegionId,
631        #[snafu(implicit)]
632        location: Location,
633    },
634
635    #[snafu(display("Region {} is truncated", region_id))]
636    RegionTruncated {
637        region_id: RegionId,
638        #[snafu(implicit)]
639        location: Location,
640    },
641
642    #[snafu(display(
643        "Engine write buffer is full, rejecting write requests of region {}",
644        region_id,
645    ))]
646    RejectWrite {
647        region_id: RegionId,
648        #[snafu(implicit)]
649        location: Location,
650    },
651
652    #[snafu(display("Region {} rejects writes while in staging", region_id))]
653    RejectWriteInStaging {
654        region_id: RegionId,
655        #[snafu(implicit)]
656        location: Location,
657    },
658
659    #[snafu(display("Region {} does not hold a valid leader lease", region_id))]
660    RegionLeaseExpired {
661        region_id: RegionId,
662        #[snafu(implicit)]
663        location: Location,
664    },
665
666    #[snafu(display(
667        "Region {} lost its leader lease while writing to the WAL, the write may have been applied",
668        region_id
669    ))]
670    UnacknowledgedWrite {
671        region_id: RegionId,
672        #[snafu(implicit)]
673        location: Location,
674    },
675
676    #[snafu(display("Failed to compact region {}", region_id))]
677    CompactRegion {
678        region_id: RegionId,
679        source: Arc<Error>,
680        #[snafu(implicit)]
681        location: Location,
682    },
683
684    #[snafu(display("Failed to edit region {}", region_id))]
685    EditRegion {
686        region_id: RegionId,
687        source: Arc<Error>,
688        #[snafu(implicit)]
689        location: Location,
690    },
691
692    #[snafu(display(
693        "Failed to compat readers for region {}, reason: {}",
694        region_id,
695        reason,
696    ))]
697    CompatReader {
698        region_id: RegionId,
699        reason: String,
700        #[snafu(implicit)]
701        location: Location,
702    },
703
704    #[snafu(display("Invalue region req"))]
705    InvalidRegionRequest {
706        source: store_api::metadata::MetadataError,
707        #[snafu(implicit)]
708        location: Location,
709    },
710
711    #[snafu(display(
712        "Region {} is in {:?} state, which does not permit manifest updates.",
713        region_id,
714        state
715    ))]
716    UpdateManifest {
717        region_id: RegionId,
718        state: RegionRoleState,
719        #[snafu(implicit)]
720        location: Location,
721    },
722
723    #[snafu(display("Region {} is in {:?} state, expect: {:?}", region_id, state, expect))]
724    RegionState {
725        region_id: RegionId,
726        state: RegionRoleState,
727        expect: RegionRoleState,
728        #[snafu(implicit)]
729        location: Location,
730    },
731
732    #[snafu(display(
733        "Partition expr version mismatch for region {}: request {}, expected {}",
734        region_id,
735        request_version,
736        expected_version
737    ))]
738    PartitionExprVersionMismatch {
739        region_id: RegionId,
740        request_version: u64,
741        expected_version: u64,
742        #[snafu(implicit)]
743        location: Location,
744    },
745
746    #[snafu(display("Invalid options"))]
747    JsonOptions {
748        #[snafu(source)]
749        error: serde_json::Error,
750        #[snafu(implicit)]
751        location: Location,
752    },
753
754    #[snafu(display(
755        "Empty region directory, region_id: {}, region_dir: {}",
756        region_id,
757        region_dir,
758    ))]
759    EmptyRegionDir {
760        region_id: RegionId,
761        region_dir: String,
762        #[snafu(implicit)]
763        location: Location,
764    },
765
766    #[snafu(display("Empty manifest directory, manifest_dir: {}", manifest_dir,))]
767    EmptyManifestDir {
768        manifest_dir: String,
769        #[snafu(implicit)]
770        location: Location,
771    },
772
773    #[snafu(display("Column not found, column: {column}"))]
774    ColumnNotFound {
775        column: String,
776        #[snafu(implicit)]
777        location: Location,
778    },
779
780    #[snafu(display("Failed to build index applier"))]
781    BuildIndexApplier {
782        source: index::inverted_index::error::Error,
783        #[snafu(implicit)]
784        location: Location,
785    },
786
787    #[snafu(display("Failed to build index asynchronously in region {}", region_id))]
788    BuildIndexAsync {
789        region_id: RegionId,
790        source: Arc<Error>,
791        #[snafu(implicit)]
792        location: Location,
793    },
794
795    #[snafu(display(
796        "Index build did not complete for files {:?} in region {}",
797        file_ids,
798        region_id
799    ))]
800    IndexBuildIncomplete {
801        region_id: RegionId,
802        file_ids: Vec<FileId>,
803        #[snafu(implicit)]
804        location: Location,
805    },
806
807    #[snafu(display("Failed to convert value"))]
808    ConvertValue {
809        source: datatypes::error::Error,
810        #[snafu(implicit)]
811        location: Location,
812    },
813
814    #[snafu(display("Failed to apply inverted index"))]
815    ApplyInvertedIndex {
816        source: index::inverted_index::error::Error,
817        #[snafu(implicit)]
818        location: Location,
819    },
820
821    #[snafu(display("Failed to apply bloom filter index"))]
822    ApplyBloomFilterIndex {
823        source: index::bloom_filter::error::Error,
824        #[snafu(implicit)]
825        location: Location,
826    },
827
828    #[snafu(display("Failed to push index value"))]
829    PushIndexValue {
830        source: index::inverted_index::error::Error,
831        #[snafu(implicit)]
832        location: Location,
833    },
834
835    #[snafu(display("Failed to write index completely"))]
836    IndexFinish {
837        source: index::inverted_index::error::Error,
838        #[snafu(implicit)]
839        location: Location,
840    },
841
842    #[snafu(display("Operate on aborted index"))]
843    OperateAbortedIndex {
844        #[snafu(implicit)]
845        location: Location,
846    },
847
848    #[snafu(display("Failed to read puffin blob"))]
849    PuffinReadBlob {
850        source: puffin::error::Error,
851        #[snafu(implicit)]
852        location: Location,
853    },
854
855    #[snafu(display("Failed to add blob to puffin file"))]
856    PuffinAddBlob {
857        source: puffin::error::Error,
858        #[snafu(implicit)]
859        location: Location,
860    },
861
862    #[snafu(display("Failed to clean dir {dir}"))]
863    CleanDir {
864        dir: String,
865        #[snafu(source)]
866        error: std::io::Error,
867        #[snafu(implicit)]
868        location: Location,
869    },
870
871    #[snafu(display("Invalid config, {reason}"))]
872    InvalidConfig {
873        reason: String,
874        #[snafu(implicit)]
875        location: Location,
876    },
877
878    #[snafu(display(
879        "Stale log entry found during replay, region: {}, flushed: {}, replayed: {}",
880        region_id,
881        flushed_entry_id,
882        unexpected_entry_id
883    ))]
884    StaleLogEntry {
885        region_id: RegionId,
886        flushed_entry_id: u64,
887        unexpected_entry_id: u64,
888    },
889
890    #[snafu(display(
891        "Failed to download file, region_id: {}, file_id: {}, file_type: {:?}",
892        region_id,
893        file_id,
894        file_type,
895    ))]
896    Download {
897        region_id: RegionId,
898        file_id: FileId,
899        file_type: FileType,
900        #[snafu(source)]
901        error: std::io::Error,
902        #[snafu(implicit)]
903        location: Location,
904    },
905
906    #[snafu(display(
907        "Failed to upload file, region_id: {}, file_id: {}, file_type: {:?}",
908        region_id,
909        file_id,
910        file_type,
911    ))]
912    Upload {
913        region_id: RegionId,
914        file_id: FileId,
915        file_type: FileType,
916        #[snafu(source)]
917        error: std::io::Error,
918        #[snafu(implicit)]
919        location: Location,
920    },
921
922    #[snafu(display("Failed to create directory {}", dir))]
923    CreateDir {
924        dir: String,
925        #[snafu(source)]
926        error: std::io::Error,
927    },
928
929    #[snafu(display("Record batch error"))]
930    RecordBatch {
931        source: common_recordbatch::error::Error,
932        #[snafu(implicit)]
933        location: Location,
934    },
935
936    #[snafu(display("BiErrors, first: {first}, second: {second}"))]
937    BiErrors {
938        first: Box<Error>,
939        second: Box<Error>,
940        #[snafu(implicit)]
941        location: Location,
942    },
943
944    #[snafu(display("Encode null value"))]
945    IndexEncodeNull {
946        #[snafu(implicit)]
947        location: Location,
948    },
949
950    #[snafu(display("Failed to encode memtable to Parquet bytes"))]
951    EncodeMemtable {
952        #[snafu(source)]
953        error: parquet::errors::ParquetError,
954        #[snafu(implicit)]
955        location: Location,
956    },
957
958    #[snafu(display("Partition {} out of range, {} in total", given, all))]
959    PartitionOutOfRange {
960        given: usize,
961        all: usize,
962        #[snafu(implicit)]
963        location: Location,
964    },
965
966    #[snafu(display("Failed to iter data part"))]
967    ReadDataPart {
968        #[snafu(implicit)]
969        location: Location,
970        #[snafu(source)]
971        error: parquet::errors::ParquetError,
972    },
973
974    #[snafu(display("Failed to read row group in memtable"))]
975    DecodeArrowRowGroup {
976        #[snafu(source)]
977        error: ArrowError,
978        #[snafu(implicit)]
979        location: Location,
980    },
981
982    #[snafu(display("Invalid region options, {}", reason))]
983    InvalidRegionOptions {
984        reason: String,
985        #[snafu(implicit)]
986        location: Location,
987    },
988
989    #[snafu(display("checksum mismatch (actual: {}, expected: {})", actual, expected))]
990    ChecksumMismatch { actual: u32, expected: u32 },
991
992    #[snafu(display(
993        "No checkpoint found, region: {}, last_version: {}",
994        region_id,
995        last_version
996    ))]
997    NoCheckpoint {
998        region_id: RegionId,
999        last_version: ManifestVersion,
1000        #[snafu(implicit)]
1001        location: Location,
1002    },
1003
1004    #[snafu(display(
1005        "No manifests found in range: [{}..{}), region: {}, last_version: {}",
1006        start_version,
1007        end_version,
1008        region_id,
1009        last_version
1010    ))]
1011    NoManifests {
1012        region_id: RegionId,
1013        start_version: ManifestVersion,
1014        end_version: ManifestVersion,
1015        last_version: ManifestVersion,
1016        #[snafu(implicit)]
1017        location: Location,
1018    },
1019
1020    #[snafu(display(
1021        "Failed to install manifest to {}, region: {}, available manifest version: {}, last version: {}",
1022        target_version,
1023        region_id,
1024        available_version,
1025        last_version
1026    ))]
1027    InstallManifestTo {
1028        region_id: RegionId,
1029        target_version: ManifestVersion,
1030        available_version: ManifestVersion,
1031        #[snafu(implicit)]
1032        location: Location,
1033        last_version: ManifestVersion,
1034    },
1035
1036    #[snafu(display("Region {} is stopped", region_id))]
1037    RegionStopped {
1038        region_id: RegionId,
1039        #[snafu(implicit)]
1040        location: Location,
1041    },
1042
1043    #[snafu(display(
1044        "Time range predicate overflows, timestamp: {:?}, target unit: {}",
1045        timestamp,
1046        unit
1047    ))]
1048    TimeRangePredicateOverflow {
1049        timestamp: Timestamp,
1050        unit: TimeUnit,
1051        #[snafu(implicit)]
1052        location: Location,
1053    },
1054
1055    #[snafu(display("Failed to open region"))]
1056    OpenRegion {
1057        #[snafu(implicit)]
1058        location: Location,
1059        source: Arc<Error>,
1060    },
1061
1062    #[snafu(display(
1063        "Region {} does not satisfy requirement '{}': {}",
1064        region_id,
1065        requirement,
1066        reason
1067    ))]
1068    RegionRequirement {
1069        region_id: RegionId,
1070        requirement: &'static str,
1071        reason: &'static str,
1072        #[snafu(implicit)]
1073        location: Location,
1074    },
1075
1076    #[snafu(display("Failed to parse job id"))]
1077    ParseJobId {
1078        #[snafu(implicit)]
1079        location: Location,
1080        #[snafu(source)]
1081        error: uuid::Error,
1082    },
1083
1084    #[snafu(display("Operation is not supported: {}", err_msg))]
1085    UnsupportedOperation {
1086        err_msg: String,
1087        #[snafu(implicit)]
1088        location: Location,
1089    },
1090
1091    #[snafu(display(
1092        "Failed to remotely compact region {} by job {:?} due to {}",
1093        region_id,
1094        job_id,
1095        reason
1096    ))]
1097    RemoteCompaction {
1098        region_id: RegionId,
1099        job_id: Option<JobId>,
1100        reason: String,
1101        #[snafu(implicit)]
1102        location: Location,
1103    },
1104
1105    #[snafu(display("Failed to initialize puffin stager"))]
1106    PuffinInitStager {
1107        source: puffin::error::Error,
1108        #[snafu(implicit)]
1109        location: Location,
1110    },
1111
1112    #[snafu(display("Failed to purge puffin stager"))]
1113    PuffinPurgeStager {
1114        source: puffin::error::Error,
1115        #[snafu(implicit)]
1116        location: Location,
1117    },
1118
1119    #[snafu(display("Failed to build puffin reader"))]
1120    PuffinBuildReader {
1121        source: puffin::error::Error,
1122        #[snafu(implicit)]
1123        location: Location,
1124    },
1125
1126    #[snafu(display("Failed to retrieve index options from column metadata"))]
1127    IndexOptions {
1128        #[snafu(implicit)]
1129        location: Location,
1130        source: datatypes::error::Error,
1131        column_name: String,
1132    },
1133
1134    #[snafu(display("Failed to retrieve compression options of column {column_name}"))]
1135    CompressionOptions {
1136        #[snafu(implicit)]
1137        location: Location,
1138        source: datatypes::error::Error,
1139        column_name: String,
1140    },
1141
1142    #[snafu(display("Failed to create fulltext index creator"))]
1143    CreateFulltextCreator {
1144        source: index::fulltext_index::error::Error,
1145        #[snafu(implicit)]
1146        location: Location,
1147    },
1148
1149    #[snafu(display("Failed to cast vector of {from} to {to}"))]
1150    CastVector {
1151        #[snafu(implicit)]
1152        location: Location,
1153        from: ConcreteDataType,
1154        to: ConcreteDataType,
1155        source: datatypes::error::Error,
1156    },
1157
1158    #[snafu(display("Failed to push text to fulltext index"))]
1159    FulltextPushText {
1160        source: index::fulltext_index::error::Error,
1161        #[snafu(implicit)]
1162        location: Location,
1163    },
1164
1165    #[snafu(display("Failed to finalize fulltext index creator"))]
1166    FulltextFinish {
1167        source: index::fulltext_index::error::Error,
1168        #[snafu(implicit)]
1169        location: Location,
1170    },
1171
1172    #[snafu(display("Failed to apply fulltext index"))]
1173    ApplyFulltextIndex {
1174        source: index::fulltext_index::error::Error,
1175        #[snafu(implicit)]
1176        location: Location,
1177    },
1178
1179    #[snafu(display("SST file {} does not contain valid stats info", file_path))]
1180    StatsNotPresent {
1181        file_path: String,
1182        #[snafu(implicit)]
1183        location: Location,
1184    },
1185
1186    #[snafu(display("Failed to decode stats of file {}", file_path))]
1187    DecodeStats {
1188        file_path: String,
1189        #[snafu(implicit)]
1190        location: Location,
1191    },
1192
1193    #[snafu(display("Invalid SST primary key range: {reason}"))]
1194    InvalidPrimaryKeyRange {
1195        reason: String,
1196        #[snafu(implicit)]
1197        location: Location,
1198    },
1199
1200    #[snafu(display("Failed to decode SST primary key range {endpoint} endpoint"))]
1201    DecodePrimaryKeyRange {
1202        endpoint: &'static str,
1203        source: mito_codec::error::Error,
1204        #[snafu(implicit)]
1205        location: Location,
1206    },
1207
1208    #[snafu(display("Region {} is busy", region_id))]
1209    RegionBusy {
1210        region_id: RegionId,
1211        #[snafu(implicit)]
1212        location: Location,
1213    },
1214
1215    #[snafu(display("Failed to get schema metadata"))]
1216    GetSchemaMetadata {
1217        source: common_meta::error::Error,
1218        #[snafu(implicit)]
1219        location: Location,
1220    },
1221
1222    #[snafu(display("Timeout"))]
1223    Timeout {
1224        #[snafu(source)]
1225        error: Elapsed,
1226        #[snafu(implicit)]
1227        location: Location,
1228    },
1229
1230    #[snafu(display("Failed to read file metadata"))]
1231    Metadata {
1232        #[snafu(source)]
1233        error: std::io::Error,
1234        #[snafu(implicit)]
1235        location: Location,
1236    },
1237
1238    #[snafu(display("Failed to push value to bloom filter"))]
1239    PushBloomFilterValue {
1240        source: index::bloom_filter::error::Error,
1241        #[snafu(implicit)]
1242        location: Location,
1243    },
1244
1245    #[snafu(display("Failed to finish bloom filter"))]
1246    BloomFilterFinish {
1247        source: index::bloom_filter::error::Error,
1248        #[snafu(implicit)]
1249        location: Location,
1250    },
1251
1252    #[snafu(display("Manual compaction is override by following operations."))]
1253    ManualCompactionOverride {},
1254
1255    #[snafu(display("Manual compaction is already running for region {region_id}."))]
1256    ManualCompactionAlreadyRunning { region_id: RegionId },
1257
1258    #[snafu(display("Compaction is cancelled."))]
1259    CompactionCancelled {},
1260
1261    #[snafu(display("Flush is cancelled."))]
1262    FlushCancelled {},
1263
1264    #[snafu(display("Compaction memory exhausted for region {region_id} (policy: {policy})",))]
1265    CompactionMemoryExhausted {
1266        region_id: RegionId,
1267        policy: String,
1268        #[snafu(source)]
1269        source: common_memory_manager::Error,
1270        #[snafu(implicit)]
1271        location: Location,
1272    },
1273
1274    #[snafu(display(
1275        "Incompatible WAL provider change. This is typically caused by changing WAL provider in database config file without completely cleaning existing files. Global provider: {}, region provider: {}",
1276        global,
1277        region
1278    ))]
1279    IncompatibleWalProviderChange { global: String, region: String },
1280
1281    #[snafu(display("Expected mito manifest info"))]
1282    MitoManifestInfo {
1283        #[snafu(implicit)]
1284        location: Location,
1285    },
1286
1287    #[snafu(display("Failed to scan series"))]
1288    ScanSeries {
1289        #[snafu(implicit)]
1290        location: Location,
1291        source: Arc<Error>,
1292    },
1293
1294    #[snafu(display("Partition {} scan multiple times", partition))]
1295    ScanMultiTimes {
1296        partition: usize,
1297        #[snafu(implicit)]
1298        location: Location,
1299    },
1300
1301    #[snafu(display("Invalid partition expression: {}", expr))]
1302    InvalidPartitionExpr {
1303        expr: String,
1304        #[snafu(implicit)]
1305        location: Location,
1306        source: partition::error::Error,
1307    },
1308
1309    #[snafu(display("Failed to decode bulk wal entry"))]
1310    ConvertBulkWalEntry {
1311        #[snafu(implicit)]
1312        location: Location,
1313        source: common_grpc::Error,
1314    },
1315
1316    #[snafu(display("Failed to encode"))]
1317    Encode {
1318        #[snafu(implicit)]
1319        location: Location,
1320        source: mito_codec::error::Error,
1321    },
1322
1323    #[snafu(display("Failed to decode"))]
1324    Decode {
1325        #[snafu(implicit)]
1326        location: Location,
1327        source: mito_codec::error::Error,
1328    },
1329
1330    #[snafu(display("Unexpected: {reason}"))]
1331    Unexpected {
1332        reason: String,
1333        #[snafu(implicit)]
1334        location: Location,
1335    },
1336
1337    #[cfg(feature = "enterprise")]
1338    #[snafu(display("Failed to scan external range"))]
1339    ScanExternalRange {
1340        source: BoxedError,
1341        #[snafu(implicit)]
1342        location: Location,
1343    },
1344
1345    #[snafu(display(
1346        "Inconsistent timestamp column length, expect: {}, actual: {}",
1347        expected,
1348        actual
1349    ))]
1350    InconsistentTimestampLength {
1351        expected: usize,
1352        actual: usize,
1353        #[snafu(implicit)]
1354        location: Location,
1355    },
1356
1357    #[snafu(display(
1358        "Too many files to read concurrently: {}, max allowed: {}",
1359        actual,
1360        max
1361    ))]
1362    TooManyFilesToRead {
1363        actual: usize,
1364        max: usize,
1365        #[snafu(implicit)]
1366        location: Location,
1367    },
1368
1369    #[snafu(display("Duration out of range: {input:?}"))]
1370    DurationOutOfRange {
1371        input: std::time::Duration,
1372        #[snafu(source)]
1373        error: chrono::OutOfRangeError,
1374        #[snafu(implicit)]
1375        location: Location,
1376    },
1377
1378    #[snafu(display("GC job permit exhausted"))]
1379    TooManyGcJobs {
1380        #[snafu(implicit)]
1381        location: Location,
1382    },
1383
1384    #[snafu(display(
1385        "Staging partition expr mismatch, manifest: {:?}, request: {}",
1386        manifest_expr,
1387        request_expr
1388    ))]
1389    StagingPartitionExprMismatch {
1390        manifest_expr: Option<String>,
1391        request_expr: String,
1392        #[snafu(implicit)]
1393        location: Location,
1394    },
1395
1396    #[snafu(display(
1397        "Invalid source and target region, source: {}, target: {}",
1398        source_region_id,
1399        target_region_id
1400    ))]
1401    InvalidSourceAndTargetRegion {
1402        source_region_id: RegionId,
1403        target_region_id: RegionId,
1404        #[snafu(implicit)]
1405        location: Location,
1406    },
1407
1408    #[snafu(display("Failed to prune file"))]
1409    PruneFile {
1410        source: Arc<Error>,
1411        #[snafu(implicit)]
1412        location: Location,
1413    },
1414
1415    #[snafu(display("Failed to cast column"))]
1416    CastColumn {
1417        #[snafu(source)]
1418        error: datafusion::error::DataFusionError,
1419        #[snafu(implicit)]
1420        location: Location,
1421    },
1422
1423    #[snafu(display("Failed to generate Arrow schema from Parquet file: {}", file))]
1424    ParquetToArrowSchema {
1425        file: String,
1426        #[snafu(source)]
1427        error: parquet::errors::ParquetError,
1428        #[snafu(implicit)]
1429        location: Location,
1430    },
1431
1432    #[snafu(display(
1433        "Region {} is in {:?} state, expect: Writable, Staging or Downgrading",
1434        region_id,
1435        state
1436    ))]
1437    FlushableRegionState {
1438        region_id: RegionId,
1439        state: RegionRoleState,
1440        #[snafu(implicit)]
1441        location: Location,
1442    },
1443}
1444
1445pub type Result<T, E = Error> = std::result::Result<T, E>;
1446
1447impl Error {
1448    /// Returns true if we need to fill default value for a region.
1449    pub(crate) fn is_fill_default(&self) -> bool {
1450        matches!(self, Error::FillDefault { .. })
1451    }
1452
1453    /// Returns true if the file is not found on the object store.
1454    pub(crate) fn is_object_not_found(&self) -> bool {
1455        match self {
1456            Error::OpenDal { error, .. } => error.kind() == ErrorKind::NotFound,
1457            Error::ManifestDeltaNotFound { .. } => true,
1458            _ => false,
1459        }
1460    }
1461
1462    /// Returns whether a failed manifest update may have been persisted.
1463    ///
1464    /// Unknown errors are treated conservatively because deleting output SSTs after an ambiguous
1465    /// manifest write could leave the manifest referencing missing files. The retry layer returns
1466    /// the last attempt's error, so an error that can follow a stored attempt (throttling, or a 403
1467    /// from a revoked key, a changed bucket policy or clock skew) never proves nothing was written.
1468    pub(crate) fn may_have_persisted_manifest_update(&self) -> bool {
1469        match self {
1470            Error::UpdateManifest { .. }
1471            | Error::RegionState { .. }
1472            | Error::RegionTruncated { .. }
1473            | Error::RegionStopped { .. }
1474            | Error::SerdeJson { .. }
1475            | Error::CompressObject { .. } => false,
1476            Error::OpenDal { error, .. } => !matches!(
1477                error.kind(),
1478                ErrorKind::Unsupported
1479                    | ErrorKind::ConfigInvalid
1480                    | ErrorKind::NotFound
1481                    | ErrorKind::IsADirectory
1482                    | ErrorKind::NotADirectory
1483                    | ErrorKind::AlreadyExists
1484                    | ErrorKind::IsSameFile
1485                    | ErrorKind::ConditionNotMatch
1486                    | ErrorKind::RangeNotSatisfied
1487            ),
1488            _ => true,
1489        }
1490    }
1491}
1492
1493impl ErrorExt for Error {
1494    fn status_code(&self) -> StatusCode {
1495        use Error::*;
1496
1497        match self {
1498            DataTypeMismatch { source, .. } => source.status_code(),
1499            OpenDal { .. } | ManifestDeltaNotFound { .. } | ReadParquet { .. } => {
1500                StatusCode::StorageUnavailable
1501            }
1502            WriteWal { source, .. }
1503            | ReadWal { source, .. }
1504            | DeleteWal { source, .. }
1505            | WaitWalDurable { source, .. }
1506            | CheckWalHistory { source, .. } => source.status_code(),
1507            CompressObject { .. }
1508            | DecompressObject { .. }
1509            | SerdeJson { .. }
1510            | Utf8 { .. }
1511            | NewRecordBatch { .. }
1512            | RegionCorrupted { .. }
1513            | InconsistentFile { .. }
1514            | CreateDefault { .. }
1515            | InvalidParquet { .. }
1516            | OperateAbortedIndex { .. }
1517            | IndexEncodeNull { .. }
1518            | NoCheckpoint { .. }
1519            | NoManifests { .. }
1520            | FilesLost { .. }
1521            | InstallManifestTo { .. }
1522            | Unexpected { .. }
1523            | SerializeColumnMetadata { .. }
1524            | SerializeManifest { .. }
1525            | StagingPartitionExprMismatch { .. } => StatusCode::Unexpected,
1526
1527            RegionNotFound { .. } => StatusCode::RegionNotFound,
1528            ObjectStoreNotFound { .. }
1529            | InvalidScanIndex { .. }
1530            | InvalidMeta { .. }
1531            | InvalidRequest { .. }
1532            | PartitionExprVersionMismatch { .. }
1533            | FillDefault { .. }
1534            | ConvertColumnDataType { .. }
1535            | ColumnNotFound { .. }
1536            | InvalidMetadata { .. }
1537            | InvalidRegionOptions { .. }
1538            | InvalidWalReadRequest { .. }
1539            | PartitionOutOfRange { .. }
1540            | ParseJobId { .. }
1541            | DurationOutOfRange { .. }
1542            | MissingOldManifest { .. }
1543            | MissingNewManifest { .. }
1544            | MissingManifest { .. }
1545            | NoOldManifests { .. }
1546            | MissingPartitionExpr { .. }
1547            | SerializePartitionExpr { .. }
1548            | InvalidSourceAndTargetRegion { .. } => StatusCode::InvalidArguments,
1549
1550            IncrementalQueryStale { .. } | SnapshotFenceStale { .. } => StatusCode::RequestOutdated,
1551
1552            SequenceRangeUnsupported { .. } => StatusCode::Unsupported,
1553
1554            RegionSequenceDomainBroken { .. }
1555            | RegionMetadataNotFound { .. }
1556            | Join { .. }
1557            | WorkerStopped { .. }
1558            | Recv { .. }
1559            | DecodeWal { .. }
1560            | ComputeArrow { .. }
1561            | EvalPartitionFilter { .. }
1562            | MergeCandidateSeries { .. }
1563            | BiErrors { .. }
1564            | StopScheduler { .. }
1565            | ComputeVector { .. }
1566            | EncodeMemtable { .. }
1567            | CreateDir { .. }
1568            | ReadDataPart { .. }
1569            | BuildEntry { .. }
1570            | Metadata { .. }
1571            | CastColumn { .. }
1572            | MitoManifestInfo { .. }
1573            | ParquetToArrowSchema { .. } => StatusCode::Internal,
1574
1575            FetchManifests { source, .. } => source.status_code(),
1576
1577            OpenRegion { source, .. } => source.status_code(),
1578
1579            WriteParquet { .. } => StatusCode::StorageUnavailable,
1580            WriteGroup { source, .. } => source.status_code(),
1581            InvalidBatch { .. } => StatusCode::InvalidArguments,
1582            InvalidRecordBatch { .. } => StatusCode::InvalidArguments,
1583            ConvertVector { source, .. } => source.status_code(),
1584
1585            PrimaryKeyLengthMismatch { .. } => StatusCode::InvalidArguments,
1586            InvalidSender { .. } => StatusCode::InvalidArguments,
1587            InvalidSchedulerState { .. } => StatusCode::InvalidArguments,
1588            RegionRequirement { .. } => StatusCode::InvalidArguments,
1589            DeleteSsts { .. } | DeleteIndex { .. } | DeleteIndexes { .. } => {
1590                StatusCode::StorageUnavailable
1591            }
1592            FlushRegion { source, .. } | BuildIndexAsync { source, .. } => source.status_code(),
1593            IndexBuildIncomplete { .. } => StatusCode::IllegalState,
1594            RegionDropped { .. } => StatusCode::Cancelled,
1595            RegionClosed { .. } => StatusCode::Cancelled,
1596            StaleCompactionExecution { .. } => StatusCode::Cancelled,
1597            RegionTruncated { .. } => StatusCode::Cancelled,
1598            RejectWrite { .. } | RejectWriteInStaging { .. } => StatusCode::StorageUnavailable,
1599            CompactRegion { source, .. } => source.status_code(),
1600            EditRegion { source, .. } => source.status_code(),
1601            CompatReader { .. } => StatusCode::Unexpected,
1602            InvalidRegionRequest { source, .. } => source.status_code(),
1603            RegionState { .. } | UpdateManifest { .. } => StatusCode::RegionNotReady,
1604            RegionLeaseExpired { .. } | UnacknowledgedWrite { .. } => StatusCode::RegionNotReady,
1605            JsonOptions { .. } => StatusCode::InvalidArguments,
1606            EmptyRegionDir { .. } | EmptyManifestDir { .. } => StatusCode::RegionNotFound,
1607            ConvertValue { source, .. } => source.status_code(),
1608            ApplyBloomFilterIndex { source, .. } => source.status_code(),
1609            InvalidPartitionExpr { source, .. } => source.status_code(),
1610            BuildIndexApplier { source, .. }
1611            | PushIndexValue { source, .. }
1612            | ApplyInvertedIndex { source, .. }
1613            | IndexFinish { source, .. } => source.status_code(),
1614            PuffinReadBlob { source, .. }
1615            | PuffinAddBlob { source, .. }
1616            | PuffinInitStager { source, .. }
1617            | PuffinBuildReader { source, .. }
1618            | PuffinPurgeStager { source, .. } => source.status_code(),
1619            CleanDir { .. } => StatusCode::Unexpected,
1620            InvalidConfig { .. } => StatusCode::InvalidArguments,
1621            StaleLogEntry { .. }
1622            | InvalidNativeHistogramSubfield { .. }
1623            | InvalidNativeHistogramFieldId { .. } => StatusCode::Unexpected,
1624
1625            External { source, .. } => source.status_code(),
1626
1627            RecordBatch { source, .. } => source.status_code(),
1628
1629            Download { .. } | Upload { .. } => StatusCode::StorageUnavailable,
1630            ChecksumMismatch { .. } => StatusCode::Unexpected,
1631            RegionStopped { .. } => StatusCode::RegionNotReady,
1632            TimeRangePredicateOverflow { .. } => StatusCode::InvalidArguments,
1633            UnsupportedOperation { .. } => StatusCode::Unsupported,
1634            RemoteCompaction { .. } => StatusCode::Unexpected,
1635
1636            IndexOptions { source, .. } => source.status_code(),
1637            CompressionOptions { source, .. } => source.status_code(),
1638            CreateFulltextCreator { source, .. } => source.status_code(),
1639            CastVector { source, .. } => source.status_code(),
1640            FulltextPushText { source, .. }
1641            | FulltextFinish { source, .. }
1642            | ApplyFulltextIndex { source, .. } => source.status_code(),
1643            DecodeStats { .. }
1644            | StatsNotPresent { .. }
1645            | InvalidPrimaryKeyRange { .. }
1646            | DecodePrimaryKeyRange { .. } => StatusCode::Internal,
1647            RegionBusy { .. } => StatusCode::RegionBusy,
1648            GetSchemaMetadata { source, .. } => source.status_code(),
1649            Timeout { .. } => StatusCode::Cancelled,
1650
1651            DecodeArrowRowGroup { .. } => StatusCode::Internal,
1652
1653            PushBloomFilterValue { source, .. } | BloomFilterFinish { source, .. } => {
1654                source.status_code()
1655            }
1656
1657            ManualCompactionOverride {} | CompactionCancelled {} | FlushCancelled {} => {
1658                StatusCode::Cancelled
1659            }
1660
1661            // A concurrent manual compaction fails fast instead of being queued;
1662            // the conflict is reported to the caller, which decides whether to
1663            // issue a new request after the running one finishes.
1664            ManualCompactionAlreadyRunning { .. } => StatusCode::RegionBusy,
1665
1666            CompactionMemoryExhausted { source, .. } => source.status_code(),
1667
1668            IncompatibleWalProviderChange { .. } => StatusCode::InvalidArguments,
1669
1670            ScanSeries { source, .. } => source.status_code(),
1671
1672            ScanMultiTimes { .. } => StatusCode::InvalidArguments,
1673            ConvertBulkWalEntry { source, .. } => source.status_code(),
1674
1675            Encode { source, .. } | Decode { source, .. } => source.status_code(),
1676
1677            #[cfg(feature = "enterprise")]
1678            ScanExternalRange { source, .. } => source.status_code(),
1679
1680            InconsistentTimestampLength { .. } => StatusCode::InvalidArguments,
1681
1682            TooManyFilesToRead { .. } | TooManyGcJobs { .. } => StatusCode::RateLimited,
1683
1684            PruneFile { source, .. } => source.status_code(),
1685
1686            FlushableRegionState { .. } => StatusCode::RegionNotReady,
1687        }
1688    }
1689
1690    fn as_any(&self) -> &dyn Any {
1691        self
1692    }
1693
1694    fn retry_hint(&self) -> RetryHint {
1695        use Error::*;
1696
1697        match self {
1698            PartitionExprVersionMismatch { .. }
1699            | RegionNotFound { .. }
1700            | RegionClosed { .. }
1701            | RejectWriteInStaging { .. }
1702            | RegionLeaseExpired { .. }
1703            | UnacknowledgedWrite { .. }
1704            | RegionState {
1705                state: RegionRoleState::Follower,
1706                ..
1707            } => RetryHint::RetryAfterRouteRefresh,
1708            ReadParquet { .. }
1709            | WriteParquet { .. }
1710            | RejectWrite { .. }
1711            | Download { .. }
1712            | Upload { .. }
1713            | RegionState { .. }
1714            | UpdateManifest { .. }
1715            | RegionStopped { .. }
1716            | RegionBusy { .. }
1717            | ManualCompactionAlreadyRunning { .. }
1718            | FlushableRegionState { .. }
1719            | ManifestDeltaNotFound { .. } => RetryHint::Retryable,
1720
1721            OpenDal { error, .. }
1722            | DeleteSsts { error, .. }
1723            | DeleteIndex { error, .. }
1724            | DeleteIndexes { error, .. } => retry_hint_from_opendal_error(error),
1725
1726            WriteWal { source, .. }
1727            | ReadWal { source, .. }
1728            | DeleteWal { source, .. }
1729            | WaitWalDurable { source, .. }
1730            | CheckWalHistory { source, .. }
1731            | FetchManifests { source, .. }
1732            | External { source, .. } => source.retry_hint(),
1733
1734            OpenRegion { source, .. }
1735            | WriteGroup { source, .. }
1736            | FlushRegion { source, .. }
1737            | BuildIndexAsync { source, .. }
1738            | CompactRegion { source, .. }
1739            | EditRegion { source, .. }
1740            | ScanSeries { source, .. }
1741            | PruneFile { source, .. } => source.retry_hint(),
1742
1743            DataTypeMismatch { source, .. }
1744            | ConvertVector { source, .. }
1745            | ConvertValue { source, .. }
1746            | IndexOptions { source, .. }
1747            | CompressionOptions { source, .. }
1748            | CastVector { source, .. } => source.retry_hint(),
1749
1750            BuildIndexApplier { source, .. }
1751            | PushIndexValue { source, .. }
1752            | ApplyInvertedIndex { source, .. }
1753            | IndexFinish { source, .. } => source.retry_hint(),
1754
1755            ApplyBloomFilterIndex { source, .. }
1756            | PushBloomFilterValue { source, .. }
1757            | BloomFilterFinish { source, .. } => source.retry_hint(),
1758
1759            PuffinReadBlob { source, .. }
1760            | PuffinAddBlob { source, .. }
1761            | PuffinInitStager { source, .. }
1762            | PuffinBuildReader { source, .. }
1763            | PuffinPurgeStager { source, .. } => source.retry_hint(),
1764
1765            CreateFulltextCreator { source, .. }
1766            | FulltextPushText { source, .. }
1767            | FulltextFinish { source, .. }
1768            | ApplyFulltextIndex { source, .. } => source.retry_hint(),
1769
1770            InvalidRegionRequest { source, .. } => source.retry_hint(),
1771            InvalidPartitionExpr { source, .. } => source.retry_hint(),
1772            RecordBatch { source, .. } => source.retry_hint(),
1773            GetSchemaMetadata { source, .. } => source.retry_hint(),
1774            CompactionMemoryExhausted { source, .. } => source.retry_hint(),
1775            ConvertBulkWalEntry { source, .. } => source.retry_hint(),
1776            Encode { source, .. } | Decode { source, .. } => source.retry_hint(),
1777
1778            #[cfg(feature = "enterprise")]
1779            ScanExternalRange { source, .. } => source.retry_hint(),
1780
1781            _ => RetryHint::NonRetryable,
1782        }
1783    }
1784}
1785
1786#[cfg(test)]
1787mod tests {
1788    use snafu::IntoError;
1789
1790    use super::*;
1791    use crate::region::RegionLeaderState;
1792
1793    #[test]
1794    fn test_partition_version_mismatch_retry_hint() {
1795        let error = PartitionExprVersionMismatchSnafu {
1796            region_id: RegionId::new(1, 1),
1797            request_version: 1u64,
1798            expected_version: 2u64,
1799        }
1800        .build();
1801        assert_eq!(error.status_code(), StatusCode::InvalidArguments);
1802        assert_eq!(error.retry_hint(), RetryHint::RetryAfterRouteRefresh);
1803    }
1804
1805    #[test]
1806    fn test_region_state_retry_hint() {
1807        let region_id = RegionId::new(1, 1);
1808        let writable = RegionRoleState::Leader(RegionLeaderState::Writable);
1809        let follower_error = RegionStateSnafu {
1810            region_id,
1811            state: RegionRoleState::Follower,
1812            expect: writable,
1813        }
1814        .build();
1815        assert_eq!(
1816            follower_error.retry_hint(),
1817            RetryHint::RetryAfterRouteRefresh
1818        );
1819        assert_eq!(follower_error.status_code(), StatusCode::RegionNotReady);
1820        let leader_error = RegionStateSnafu {
1821            region_id,
1822            state: writable,
1823            expect: RegionRoleState::Follower,
1824        }
1825        .build();
1826        assert_eq!(leader_error.retry_hint(), RetryHint::Retryable);
1827        for (error, status, hint) in [
1828            (
1829                RejectWriteSnafu { region_id }.build(),
1830                StatusCode::StorageUnavailable,
1831                RetryHint::Retryable,
1832            ),
1833            (
1834                RejectWriteInStagingSnafu { region_id }.build(),
1835                StatusCode::StorageUnavailable,
1836                RetryHint::RetryAfterRouteRefresh,
1837            ),
1838            (
1839                RegionNotFoundSnafu { region_id }.build(),
1840                StatusCode::RegionNotFound,
1841                RetryHint::RetryAfterRouteRefresh,
1842            ),
1843            (
1844                RegionClosedSnafu { region_id }.build(),
1845                StatusCode::Cancelled,
1846                RetryHint::RetryAfterRouteRefresh,
1847            ),
1848            (
1849                RegionDroppedSnafu { region_id }.build(),
1850                StatusCode::Cancelled,
1851                RetryHint::NonRetryable,
1852            ),
1853            (
1854                RegionTruncatedSnafu { region_id }.build(),
1855                StatusCode::Cancelled,
1856                RetryHint::NonRetryable,
1857            ),
1858        ] {
1859            assert_eq!(error.status_code(), status);
1860            assert_eq!(error.retry_hint(), hint);
1861        }
1862    }
1863
1864    #[test]
1865    fn test_manifest_update_persistence() {
1866        let rejected_kinds = [
1867            ErrorKind::Unsupported,
1868            ErrorKind::ConfigInvalid,
1869            ErrorKind::NotFound,
1870            ErrorKind::IsADirectory,
1871            ErrorKind::NotADirectory,
1872            ErrorKind::AlreadyExists,
1873            ErrorKind::IsSameFile,
1874            ErrorKind::ConditionNotMatch,
1875            ErrorKind::RangeNotSatisfied,
1876        ];
1877        for kind in rejected_kinds {
1878            let error = OpenDalSnafu {}.into_error(object_store::Error::new(kind, "test"));
1879            assert!(
1880                !error.may_have_persisted_manifest_update(),
1881                "error kind {kind:?} should prove the manifest was not persisted"
1882            );
1883        }
1884
1885        // The last retry can fail with these kinds even though an earlier attempt was stored.
1886        for kind in [
1887            ErrorKind::Unexpected,
1888            ErrorKind::RateLimited,
1889            ErrorKind::PermissionDenied,
1890        ] {
1891            let error = OpenDalSnafu {}.into_error(object_store::Error::new(kind, "test"));
1892            assert!(
1893                error.may_have_persisted_manifest_update(),
1894                "error kind {kind:?} should not prove the manifest was not persisted"
1895            );
1896        }
1897
1898        let region_id = RegionId::new(1, 1);
1899        let error = RegionTruncatedSnafu { region_id }.build();
1900        assert!(!error.may_have_persisted_manifest_update());
1901
1902        // This can be raised while applying an edit after its manifest file was saved.
1903        let error = RegionMetadataNotFoundSnafu {}.build();
1904        assert!(error.may_have_persisted_manifest_update());
1905    }
1906}
mito2/error.rs

mito2/
error.rs
