mito2/
error.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::any::Any;
16use std::sync::Arc;
17
18use common_datasource::compression::CompressionType;
19use common_error::ext::{BoxedError, ErrorExt, RetryHint};
20use common_error::status_code::StatusCode;
21use common_macro::stack_trace_debug;
22use common_memory_manager;
23use common_runtime::JoinError;
24use common_time::Timestamp;
25use common_time::timestamp::TimeUnit;
26use datatypes::arrow::error::ArrowError;
27use datatypes::prelude::ConcreteDataType;
28use object_store::ErrorKind;
29use object_store::error::retry_hint_from_opendal_error;
30use partition::error::Error as PartitionError;
31use prost::DecodeError;
32use snafu::{Location, Snafu};
33use store_api::ManifestVersion;
34use store_api::logstore::provider::Provider;
35use store_api::storage::{FileId, RegionId};
36use tokio::time::error::Elapsed;
37
38use crate::cache::file_cache::FileType;
39use crate::region::RegionRoleState;
40use crate::schedule::remote_job_scheduler::JobId;
41use crate::worker::WorkerId;
42
43#[derive(Snafu)]
44#[snafu(visibility(pub))]
45#[stack_trace_debug]
46pub enum Error {
47    #[snafu(display("Unexpected data type"))]
48    DataTypeMismatch {
49        source: datatypes::error::Error,
50        #[snafu(implicit)]
51        location: Location,
52    },
53
54    #[snafu(display("External error, context: {}", context))]
55    External {
56        source: BoxedError,
57        context: String,
58        #[snafu(implicit)]
59        location: Location,
60    },
61
62    #[snafu(display("OpenDAL operator failed"))]
63    OpenDal {
64        #[snafu(implicit)]
65        location: Location,
66        #[snafu(source)]
67        error: object_store::Error,
68    },
69
70    #[snafu(display(
71        "Manifest delta {} disappeared after it was listed, path: {}",
72        version,
73        path
74    ))]
75    ManifestDeltaNotFound {
76        version: ManifestVersion,
77        path: String,
78        #[snafu(source)]
79        error: object_store::Error,
80        #[snafu(implicit)]
81        location: Location,
82    },
83
84    #[snafu(display("Fail to compress object by {}, path: {}", compress_type, path))]
85    CompressObject {
86        compress_type: CompressionType,
87        path: String,
88        #[snafu(source)]
89        error: std::io::Error,
90    },
91
92    #[snafu(display("Fail to decompress object by {}, path: {}", compress_type, path))]
93    DecompressObject {
94        compress_type: CompressionType,
95        path: String,
96        #[snafu(source)]
97        error: std::io::Error,
98    },
99
100    #[snafu(display("Failed to ser/de json object"))]
101    SerdeJson {
102        #[snafu(implicit)]
103        location: Location,
104        #[snafu(source)]
105        error: serde_json::Error,
106    },
107
108    #[snafu(display("Failed to serialize column metadata"))]
109    SerializeColumnMetadata {
110        #[snafu(source)]
111        error: serde_json::Error,
112        #[snafu(implicit)]
113        location: Location,
114    },
115
116    #[snafu(display("Failed to serialize manifest, region_id: {}", region_id))]
117    SerializeManifest {
118        region_id: RegionId,
119        #[snafu(source)]
120        error: serde_json::Error,
121        #[snafu(implicit)]
122        location: Location,
123    },
124
125    #[snafu(display("Invalid scan index, start: {}, end: {}", start, end))]
126    InvalidScanIndex {
127        start: ManifestVersion,
128        end: ManifestVersion,
129        #[snafu(implicit)]
130        location: Location,
131    },
132
133    #[snafu(display("Invalid UTF-8 content"))]
134    Utf8 {
135        #[snafu(implicit)]
136        location: Location,
137        #[snafu(source)]
138        error: std::str::Utf8Error,
139    },
140
141    #[snafu(display("Cannot find RegionMetadata"))]
142    RegionMetadataNotFound {
143        #[snafu(implicit)]
144        location: Location,
145    },
146
147    #[snafu(display("Failed to join handle"))]
148    Join {
149        #[snafu(source)]
150        error: common_runtime::JoinError,
151        #[snafu(implicit)]
152        location: Location,
153    },
154
155    #[snafu(display("Worker {} is stopped", id))]
156    WorkerStopped {
157        id: WorkerId,
158        #[snafu(implicit)]
159        location: Location,
160    },
161
162    #[snafu(display("Failed to recv result"))]
163    Recv {
164        #[snafu(source)]
165        error: tokio::sync::oneshot::error::RecvError,
166        #[snafu(implicit)]
167        location: Location,
168    },
169
170    #[snafu(display("Invalid metadata, {}", reason))]
171    InvalidMeta {
172        reason: String,
173        #[snafu(implicit)]
174        location: Location,
175    },
176
177    #[snafu(display("Invalid region metadata"))]
178    InvalidMetadata {
179        source: store_api::metadata::MetadataError,
180        #[snafu(implicit)]
181        location: Location,
182    },
183
184    #[snafu(display("Failed to create RecordBatch from vectors"))]
185    NewRecordBatch {
186        #[snafu(implicit)]
187        location: Location,
188        #[snafu(source)]
189        error: ArrowError,
190    },
191
192    #[snafu(display("Failed to read parquet file, path: {}", path))]
193    ReadParquet {
194        path: String,
195        #[snafu(source)]
196        error: parquet::errors::ParquetError,
197        #[snafu(implicit)]
198        location: Location,
199    },
200
201    #[snafu(display("Failed to load SST metadata"))]
202    LoadSstMeta {
203        source: Arc<Error>,
204        #[snafu(implicit)]
205        location: Location,
206    },
207
208    #[snafu(display("Failed to write parquet file"))]
209    WriteParquet {
210        #[snafu(source)]
211        error: parquet::errors::ParquetError,
212        #[snafu(implicit)]
213        location: Location,
214    },
215
216    #[snafu(display(
217        "Cannot assign a stable field id to native histogram sub-field '{}' of column id {} (unknown sub-field name or derived id overflows i32)",
218        field_name,
219        column_id
220    ))]
221    InvalidNativeHistogramSubfield {
222        column_id: i32,
223        field_name: String,
224        #[snafu(implicit)]
225        location: Location,
226    },
227
228    #[snafu(display(
229        "Native histogram column '{}' has no usable PARQUET:field_id to namespace its sub-field ids (missing, malformed, or exceeds i32::MAX)",
230        field_name
231    ))]
232    InvalidNativeHistogramFieldId {
233        field_name: String,
234        #[snafu(implicit)]
235        location: Location,
236    },
237
238    #[snafu(display("Region {} not found", region_id))]
239    RegionNotFound {
240        region_id: RegionId,
241        #[snafu(implicit)]
242        location: Location,
243    },
244
245    #[snafu(display("Object store not found: {}", object_store))]
246    ObjectStoreNotFound {
247        object_store: String,
248        #[snafu(implicit)]
249        location: Location,
250    },
251
252    #[snafu(display("Region {} is corrupted, reason: {}", region_id, reason))]
253    RegionCorrupted {
254        region_id: RegionId,
255        reason: String,
256        #[snafu(implicit)]
257        location: Location,
258    },
259
260    #[snafu(display("Invalid request to region {}, reason: {}", region_id, reason))]
261    InvalidRequest {
262        region_id: RegionId,
263        reason: String,
264        #[snafu(implicit)]
265        location: Location,
266    },
267
268    #[snafu(display(
269        "STALE_CURSOR: incremental query stale, region: {}, given_seq: {}, min_readable_seq: {}, retry_hint: FALLBACK_FULL_RECOMPUTE",
270        region_id,
271        given_seq,
272        min_readable_seq
273    ))]
274    IncrementalQueryStale {
275        region_id: RegionId,
276        given_seq: u64,
277        min_readable_seq: u64,
278        #[snafu(implicit)]
279        location: Location,
280    },
281
282    #[snafu(display(
283        "STALE_SNAPSHOT_FENCE: snapshot upper bound stale, region: {}, given_seq: {}, min_enforceable_seq: {}, retry_hint: REBIND_SNAPSHOT_FENCE",
284        region_id,
285        given_seq,
286        min_enforceable_seq
287    ))]
288    SnapshotFenceStale {
289        region_id: RegionId,
290        given_seq: u64,
291        min_enforceable_seq: u64,
292        #[snafu(implicit)]
293        location: Location,
294    },
295
296    #[snafu(display(
297        "SEQUENCE_RANGE_UNSUPPORTED: exact sequence-range read unsupported, region: {}, min_seq: {}, max_seq: {}, reason: {}, retry_hint: FALLBACK_MEMTABLE_ONLY_OR_FULL_RECOMPUTE",
298        region_id,
299        min_seq,
300        max_seq,
301        reason
302    ))]
303    SequenceRangeUnsupported {
304        region_id: RegionId,
305        min_seq: u64,
306        max_seq: u64,
307        reason: String,
308        #[snafu(implicit)]
309        location: Location,
310    },
311
312    #[snafu(display(
313        "region {} is unusable for sequence reads: file {} declares region {}",
314        region_id,
315        file_id,
316        file_region_id
317    ))]
318    RegionSequenceDomainBroken {
319        region_id: RegionId,
320        file_region_id: RegionId,
321        file_id: FileId,
322        #[snafu(implicit)]
323        location: Location,
324    },
325
326    #[snafu(display("Old manifest missing for region {}", region_id))]
327    MissingOldManifest {
328        region_id: RegionId,
329        #[snafu(implicit)]
330        location: Location,
331    },
332
333    #[snafu(display("New manifest missing for region {}", region_id))]
334    MissingNewManifest {
335        region_id: RegionId,
336        #[snafu(implicit)]
337        location: Location,
338    },
339
340    #[snafu(display("Manifest missing for region {}", region_id))]
341    MissingManifest {
342        region_id: RegionId,
343        #[snafu(implicit)]
344        location: Location,
345    },
346
347    #[snafu(display("File consistency check failed for file {}: {}", file_id, reason))]
348    InconsistentFile {
349        file_id: FileId,
350        reason: String,
351        #[snafu(implicit)]
352        location: Location,
353    },
354
355    #[snafu(display("Files lost during remapping: old={}, new={}", old_count, new_count))]
356    FilesLost {
357        old_count: usize,
358        new_count: usize,
359        #[snafu(implicit)]
360        location: Location,
361    },
362
363    #[snafu(display("No old manifests provided (need at least one for template)"))]
364    NoOldManifests {
365        #[snafu(implicit)]
366        location: Location,
367    },
368
369    #[snafu(display("Failed to fetch manifests"))]
370    FetchManifests {
371        #[snafu(implicit)]
372        location: Location,
373        source: BoxedError,
374    },
375
376    #[snafu(display("Partition expression missing for region {}", region_id))]
377    MissingPartitionExpr {
378        region_id: RegionId,
379        #[snafu(implicit)]
380        location: Location,
381    },
382
383    #[snafu(display("Failed to serialize partition expression: {}", source))]
384    SerializePartitionExpr {
385        #[snafu(source)]
386        source: PartitionError,
387        #[snafu(implicit)]
388        location: Location,
389    },
390
391    #[snafu(display(
392        "Failed to convert ConcreteDataType to ColumnDataType, reason: {}",
393        reason
394    ))]
395    ConvertColumnDataType {
396        reason: String,
397        source: api::error::Error,
398        #[snafu(implicit)]
399        location: Location,
400    },
401
402    /// An error type to indicate that schema is changed and we need
403    /// to fill default values again.
404    #[snafu(display("Need to fill default value for region {}", region_id))]
405    FillDefault {
406        region_id: RegionId,
407        // The error is for internal use so we don't need a location.
408    },
409
410    #[snafu(display(
411        "Failed to create default value for column {} of region {}",
412        column,
413        region_id
414    ))]
415    CreateDefault {
416        region_id: RegionId,
417        column: String,
418        source: datatypes::Error,
419        #[snafu(implicit)]
420        location: Location,
421    },
422
423    #[snafu(display("Failed to build entry, region_id: {}", region_id))]
424    BuildEntry {
425        region_id: RegionId,
426        #[snafu(implicit)]
427        location: Location,
428        source: BoxedError,
429    },
430
431    #[snafu(display("Failed to write WAL"))]
432    WriteWal {
433        #[snafu(implicit)]
434        location: Location,
435        source: BoxedError,
436    },
437
438    #[snafu(display("Failed to read WAL, provider: {}", provider))]
439    ReadWal {
440        provider: Provider,
441        #[snafu(implicit)]
442        location: Location,
443        source: BoxedError,
444    },
445
446    #[snafu(display("Failed to decode WAL entry, region_id: {}", region_id))]
447    DecodeWal {
448        region_id: RegionId,
449        #[snafu(implicit)]
450        location: Location,
451        #[snafu(source)]
452        error: DecodeError,
453    },
454
455    #[snafu(display("Failed to delete WAL, region_id: {}", region_id))]
456    DeleteWal {
457        region_id: RegionId,
458        #[snafu(implicit)]
459        location: Location,
460        source: BoxedError,
461    },
462
463    #[snafu(display("Failed to wait for the WAL to be durable, region_id: {}", region_id))]
464    WaitWalDurable {
465        region_id: RegionId,
466        #[snafu(implicit)]
467        location: Location,
468        source: BoxedError,
469    },
470
471    #[snafu(display("Failed to check the WAL history, region_id: {}", region_id))]
472    CheckWalHistory {
473        region_id: RegionId,
474        #[snafu(implicit)]
475        location: Location,
476        source: BoxedError,
477    },
478
479    // Shared error for each writer in the write group.
480    #[snafu(display("Failed to write region"))]
481    WriteGroup { source: Arc<Error> },
482
483    #[snafu(display("Invalid parquet SST file {}, reason: {}", file, reason))]
484    InvalidParquet {
485        file: String,
486        reason: String,
487        #[snafu(implicit)]
488        location: Location,
489    },
490
491    #[snafu(display("Invalid batch, {}", reason))]
492    InvalidBatch {
493        reason: String,
494        #[snafu(implicit)]
495        location: Location,
496    },
497
498    #[snafu(display("Invalid arrow record batch, {}", reason))]
499    InvalidRecordBatch {
500        reason: String,
501        #[snafu(implicit)]
502        location: Location,
503    },
504
505    #[snafu(display("Invalid wal read request, {}", reason))]
506    InvalidWalReadRequest {
507        reason: String,
508        #[snafu(implicit)]
509        location: Location,
510    },
511
512    #[snafu(display("Failed to convert array to vector"))]
513    ConvertVector {
514        #[snafu(implicit)]
515        location: Location,
516        source: datatypes::error::Error,
517    },
518
519    #[snafu(display("Failed to compute arrow arrays"))]
520    ComputeArrow {
521        #[snafu(implicit)]
522        location: Location,
523        #[snafu(source)]
524        error: datatypes::arrow::error::ArrowError,
525    },
526
527    #[snafu(display("Failed to evaluate partition filter"))]
528    EvalPartitionFilter {
529        #[snafu(implicit)]
530        location: Location,
531        #[snafu(source)]
532        error: datafusion::error::DataFusionError,
533    },
534
535    #[snafu(display("Failed to merge candidate series"))]
536    MergeCandidateSeries {
537        #[snafu(implicit)]
538        location: Location,
539        #[snafu(source)]
540        error: datafusion::error::DataFusionError,
541    },
542
543    #[snafu(display("Failed to compute vector"))]
544    ComputeVector {
545        #[snafu(implicit)]
546        location: Location,
547        source: datatypes::error::Error,
548    },
549
550    #[snafu(display("Primary key length mismatch, expect: {}, actual: {}", expect, actual))]
551    PrimaryKeyLengthMismatch {
552        expect: usize,
553        actual: usize,
554        #[snafu(implicit)]
555        location: Location,
556    },
557
558    #[snafu(display("Invalid sender",))]
559    InvalidSender {
560        #[snafu(implicit)]
561        location: Location,
562    },
563
564    #[snafu(display("Invalid scheduler state"))]
565    InvalidSchedulerState {
566        #[snafu(implicit)]
567        location: Location,
568    },
569
570    #[snafu(display("Failed to stop scheduler"))]
571    StopScheduler {
572        #[snafu(source)]
573        error: JoinError,
574        #[snafu(implicit)]
575        location: Location,
576    },
577
578    #[snafu(display(
579        "Failed to batch delete SST files, region id: {}, file ids: {:?}",
580        region_id,
581        file_ids
582    ))]
583    DeleteSsts {
584        region_id: RegionId,
585        file_ids: Vec<FileId>,
586        #[snafu(source)]
587        error: object_store::Error,
588        #[snafu(implicit)]
589        location: Location,
590    },
591
592    #[snafu(display("Failed to delete index file, file id: {}", file_id))]
593    DeleteIndex {
594        file_id: FileId,
595        #[snafu(source)]
596        error: object_store::Error,
597        #[snafu(implicit)]
598        location: Location,
599    },
600
601    #[snafu(display("Failed to batch delete index files, file ids: {:?}", file_ids))]
602    DeleteIndexes {
603        file_ids: Vec<FileId>,
604        #[snafu(source)]
605        error: object_store::Error,
606        #[snafu(implicit)]
607        location: Location,
608    },
609
610    #[snafu(display("Failed to flush region {}", region_id))]
611    FlushRegion {
612        region_id: RegionId,
613        source: Arc<Error>,
614        #[snafu(implicit)]
615        location: Location,
616    },
617
618    #[snafu(display("Region {} is dropped", region_id))]
619    RegionDropped {
620        region_id: RegionId,
621        #[snafu(implicit)]
622        location: Location,
623    },
624
625    #[snafu(display("Region {} is closed", region_id))]
626    RegionClosed {
627        region_id: RegionId,
628        #[snafu(implicit)]
629        location: Location,
630    },
631
632    #[snafu(display(
633        "Stale compaction execution for region {}, the region may have been reopened, truncated or the compaction was superseded",
634        region_id
635    ))]
636    StaleCompactionExecution {
637        region_id: RegionId,
638        #[snafu(implicit)]
639        location: Location,
640    },
641
642    #[snafu(display("Region {} is truncated", region_id))]
643    RegionTruncated {
644        region_id: RegionId,
645        #[snafu(implicit)]
646        location: Location,
647    },
648
649    #[snafu(display(
650        "Engine write buffer is full, rejecting write requests of region {}",
651        region_id,
652    ))]
653    RejectWrite {
654        region_id: RegionId,
655        #[snafu(implicit)]
656        location: Location,
657    },
658
659    #[snafu(display("Region {} rejects writes while in staging", region_id))]
660    RejectWriteInStaging {
661        region_id: RegionId,
662        #[snafu(implicit)]
663        location: Location,
664    },
665
666    #[snafu(display("Region {} does not hold a valid leader lease", region_id))]
667    RegionLeaseExpired {
668        region_id: RegionId,
669        #[snafu(implicit)]
670        location: Location,
671    },
672
673    #[snafu(display(
674        "Region {} lost its leader lease while writing to the WAL, the write may have been applied",
675        region_id
676    ))]
677    UnacknowledgedWrite {
678        region_id: RegionId,
679        #[snafu(implicit)]
680        location: Location,
681    },
682
683    #[snafu(display("Failed to compact region {}", region_id))]
684    CompactRegion {
685        region_id: RegionId,
686        source: Arc<Error>,
687        #[snafu(implicit)]
688        location: Location,
689    },
690
691    #[snafu(display("Failed to edit region {}", region_id))]
692    EditRegion {
693        region_id: RegionId,
694        source: Arc<Error>,
695        #[snafu(implicit)]
696        location: Location,
697    },
698
699    #[snafu(display(
700        "Failed to compat readers for region {}, reason: {}",
701        region_id,
702        reason,
703    ))]
704    CompatReader {
705        region_id: RegionId,
706        reason: String,
707        #[snafu(implicit)]
708        location: Location,
709    },
710
711    #[snafu(display("Invalue region req"))]
712    InvalidRegionRequest {
713        source: store_api::metadata::MetadataError,
714        #[snafu(implicit)]
715        location: Location,
716    },
717
718    #[snafu(display(
719        "Region {} is in {:?} state, which does not permit manifest updates.",
720        region_id,
721        state
722    ))]
723    UpdateManifest {
724        region_id: RegionId,
725        state: RegionRoleState,
726        #[snafu(implicit)]
727        location: Location,
728    },
729
730    #[snafu(display("Region {} is in {:?} state, expect: {:?}", region_id, state, expect))]
731    RegionState {
732        region_id: RegionId,
733        state: RegionRoleState,
734        expect: RegionRoleState,
735        #[snafu(implicit)]
736        location: Location,
737    },
738
739    #[snafu(display(
740        "Partition expr version mismatch for region {}: request {}, expected {}",
741        region_id,
742        request_version,
743        expected_version
744    ))]
745    PartitionExprVersionMismatch {
746        region_id: RegionId,
747        request_version: u64,
748        expected_version: u64,
749        #[snafu(implicit)]
750        location: Location,
751    },
752
753    #[snafu(display("Invalid options"))]
754    JsonOptions {
755        #[snafu(source)]
756        error: serde_json::Error,
757        #[snafu(implicit)]
758        location: Location,
759    },
760
761    #[snafu(display(
762        "Empty region directory, region_id: {}, region_dir: {}",
763        region_id,
764        region_dir,
765    ))]
766    EmptyRegionDir {
767        region_id: RegionId,
768        region_dir: String,
769        #[snafu(implicit)]
770        location: Location,
771    },
772
773    #[snafu(display("Empty manifest directory, manifest_dir: {}", manifest_dir,))]
774    EmptyManifestDir {
775        manifest_dir: String,
776        #[snafu(implicit)]
777        location: Location,
778    },
779
780    #[snafu(display("Column not found, column: {column}"))]
781    ColumnNotFound {
782        column: String,
783        #[snafu(implicit)]
784        location: Location,
785    },
786
787    #[snafu(display("Failed to build index applier"))]
788    BuildIndexApplier {
789        source: index::inverted_index::error::Error,
790        #[snafu(implicit)]
791        location: Location,
792    },
793
794    #[snafu(display("Failed to build index asynchronously in region {}", region_id))]
795    BuildIndexAsync {
796        region_id: RegionId,
797        source: Arc<Error>,
798        #[snafu(implicit)]
799        location: Location,
800    },
801
802    #[snafu(display(
803        "Index build did not complete for files {:?} in region {}",
804        file_ids,
805        region_id
806    ))]
807    IndexBuildIncomplete {
808        region_id: RegionId,
809        file_ids: Vec<FileId>,
810        #[snafu(implicit)]
811        location: Location,
812    },
813
814    #[snafu(display("Failed to convert value"))]
815    ConvertValue {
816        source: datatypes::error::Error,
817        #[snafu(implicit)]
818        location: Location,
819    },
820
821    #[snafu(display("Failed to apply inverted index"))]
822    ApplyInvertedIndex {
823        source: index::inverted_index::error::Error,
824        #[snafu(implicit)]
825        location: Location,
826    },
827
828    #[snafu(display("Failed to apply bloom filter index"))]
829    ApplyBloomFilterIndex {
830        source: index::bloom_filter::error::Error,
831        #[snafu(implicit)]
832        location: Location,
833    },
834
835    #[snafu(display("Failed to push index value"))]
836    PushIndexValue {
837        source: index::inverted_index::error::Error,
838        #[snafu(implicit)]
839        location: Location,
840    },
841
842    #[snafu(display("Failed to write index completely"))]
843    IndexFinish {
844        source: index::inverted_index::error::Error,
845        #[snafu(implicit)]
846        location: Location,
847    },
848
849    #[snafu(display("Operate on aborted index"))]
850    OperateAbortedIndex {
851        #[snafu(implicit)]
852        location: Location,
853    },
854
855    #[snafu(display("Failed to read puffin blob"))]
856    PuffinReadBlob {
857        source: puffin::error::Error,
858        #[snafu(implicit)]
859        location: Location,
860    },
861
862    #[snafu(display("Failed to add blob to puffin file"))]
863    PuffinAddBlob {
864        source: puffin::error::Error,
865        #[snafu(implicit)]
866        location: Location,
867    },
868
869    #[snafu(display("Failed to clean dir {dir}"))]
870    CleanDir {
871        dir: String,
872        #[snafu(source)]
873        error: std::io::Error,
874        #[snafu(implicit)]
875        location: Location,
876    },
877
878    #[snafu(display("Invalid config, {reason}"))]
879    InvalidConfig {
880        reason: String,
881        #[snafu(implicit)]
882        location: Location,
883    },
884
885    #[snafu(display(
886        "Stale log entry found during replay, region: {}, flushed: {}, replayed: {}",
887        region_id,
888        flushed_entry_id,
889        unexpected_entry_id
890    ))]
891    StaleLogEntry {
892        region_id: RegionId,
893        flushed_entry_id: u64,
894        unexpected_entry_id: u64,
895    },
896
897    #[snafu(display(
898        "Failed to download file, region_id: {}, file_id: {}, file_type: {:?}",
899        region_id,
900        file_id,
901        file_type,
902    ))]
903    Download {
904        region_id: RegionId,
905        file_id: FileId,
906        file_type: FileType,
907        #[snafu(source)]
908        error: std::io::Error,
909        #[snafu(implicit)]
910        location: Location,
911    },
912
913    #[snafu(display(
914        "Failed to upload file, region_id: {}, file_id: {}, file_type: {:?}",
915        region_id,
916        file_id,
917        file_type,
918    ))]
919    Upload {
920        region_id: RegionId,
921        file_id: FileId,
922        file_type: FileType,
923        #[snafu(source)]
924        error: std::io::Error,
925        #[snafu(implicit)]
926        location: Location,
927    },
928
929    #[snafu(display("Failed to create directory {}", dir))]
930    CreateDir {
931        dir: String,
932        #[snafu(source)]
933        error: std::io::Error,
934    },
935
936    #[snafu(display("Record batch error"))]
937    RecordBatch {
938        source: common_recordbatch::error::Error,
939        #[snafu(implicit)]
940        location: Location,
941    },
942
943    #[snafu(display("BiErrors, first: {first}, second: {second}"))]
944    BiErrors {
945        first: Box<Error>,
946        second: Box<Error>,
947        #[snafu(implicit)]
948        location: Location,
949    },
950
951    #[snafu(display("Encode null value"))]
952    IndexEncodeNull {
953        #[snafu(implicit)]
954        location: Location,
955    },
956
957    #[snafu(display("Failed to encode memtable to Parquet bytes"))]
958    EncodeMemtable {
959        #[snafu(source)]
960        error: parquet::errors::ParquetError,
961        #[snafu(implicit)]
962        location: Location,
963    },
964
965    #[snafu(display("Partition {} out of range, {} in total", given, all))]
966    PartitionOutOfRange {
967        given: usize,
968        all: usize,
969        #[snafu(implicit)]
970        location: Location,
971    },
972
973    #[snafu(display("Failed to iter data part"))]
974    ReadDataPart {
975        #[snafu(implicit)]
976        location: Location,
977        #[snafu(source)]
978        error: parquet::errors::ParquetError,
979    },
980
981    #[snafu(display("Failed to read row group in memtable"))]
982    DecodeArrowRowGroup {
983        #[snafu(source)]
984        error: ArrowError,
985        #[snafu(implicit)]
986        location: Location,
987    },
988
989    #[snafu(display("Invalid region options, {}", reason))]
990    InvalidRegionOptions {
991        reason: String,
992        #[snafu(implicit)]
993        location: Location,
994    },
995
996    #[snafu(display("checksum mismatch (actual: {}, expected: {})", actual, expected))]
997    ChecksumMismatch { actual: u32, expected: u32 },
998
999    #[snafu(display(
1000        "No checkpoint found, region: {}, last_version: {}",
1001        region_id,
1002        last_version
1003    ))]
1004    NoCheckpoint {
1005        region_id: RegionId,
1006        last_version: ManifestVersion,
1007        #[snafu(implicit)]
1008        location: Location,
1009    },
1010
1011    #[snafu(display(
1012        "No manifests found in range: [{}..{}), region: {}, last_version: {}",
1013        start_version,
1014        end_version,
1015        region_id,
1016        last_version
1017    ))]
1018    NoManifests {
1019        region_id: RegionId,
1020        start_version: ManifestVersion,
1021        end_version: ManifestVersion,
1022        last_version: ManifestVersion,
1023        #[snafu(implicit)]
1024        location: Location,
1025    },
1026
1027    #[snafu(display(
1028        "Failed to install manifest to {}, region: {}, available manifest version: {}, last version: {}",
1029        target_version,
1030        region_id,
1031        available_version,
1032        last_version
1033    ))]
1034    InstallManifestTo {
1035        region_id: RegionId,
1036        target_version: ManifestVersion,
1037        available_version: ManifestVersion,
1038        #[snafu(implicit)]
1039        location: Location,
1040        last_version: ManifestVersion,
1041    },
1042
1043    #[snafu(display("Region {} is stopped", region_id))]
1044    RegionStopped {
1045        region_id: RegionId,
1046        #[snafu(implicit)]
1047        location: Location,
1048    },
1049
1050    #[snafu(display(
1051        "Time range predicate overflows, timestamp: {:?}, target unit: {}",
1052        timestamp,
1053        unit
1054    ))]
1055    TimeRangePredicateOverflow {
1056        timestamp: Timestamp,
1057        unit: TimeUnit,
1058        #[snafu(implicit)]
1059        location: Location,
1060    },
1061
1062    #[snafu(display("Failed to open region"))]
1063    OpenRegion {
1064        #[snafu(implicit)]
1065        location: Location,
1066        source: Arc<Error>,
1067    },
1068
1069    #[snafu(display(
1070        "Region {} does not satisfy requirement '{}': {}",
1071        region_id,
1072        requirement,
1073        reason
1074    ))]
1075    RegionRequirement {
1076        region_id: RegionId,
1077        requirement: &'static str,
1078        reason: &'static str,
1079        #[snafu(implicit)]
1080        location: Location,
1081    },
1082
1083    #[snafu(display("Failed to parse job id"))]
1084    ParseJobId {
1085        #[snafu(implicit)]
1086        location: Location,
1087        #[snafu(source)]
1088        error: uuid::Error,
1089    },
1090
1091    #[snafu(display("Operation is not supported: {}", err_msg))]
1092    UnsupportedOperation {
1093        err_msg: String,
1094        #[snafu(implicit)]
1095        location: Location,
1096    },
1097
1098    #[snafu(display(
1099        "Failed to remotely compact region {} by job {:?} due to {}",
1100        region_id,
1101        job_id,
1102        reason
1103    ))]
1104    RemoteCompaction {
1105        region_id: RegionId,
1106        job_id: Option<JobId>,
1107        reason: String,
1108        #[snafu(implicit)]
1109        location: Location,
1110    },
1111
1112    #[snafu(display("Failed to initialize puffin stager"))]
1113    PuffinInitStager {
1114        source: puffin::error::Error,
1115        #[snafu(implicit)]
1116        location: Location,
1117    },
1118
1119    #[snafu(display("Failed to purge puffin stager"))]
1120    PuffinPurgeStager {
1121        source: puffin::error::Error,
1122        #[snafu(implicit)]
1123        location: Location,
1124    },
1125
1126    #[snafu(display("Failed to build puffin reader"))]
1127    PuffinBuildReader {
1128        source: puffin::error::Error,
1129        #[snafu(implicit)]
1130        location: Location,
1131    },
1132
1133    #[snafu(display("Failed to retrieve index options from column metadata"))]
1134    IndexOptions {
1135        #[snafu(implicit)]
1136        location: Location,
1137        source: datatypes::error::Error,
1138        column_name: String,
1139    },
1140
1141    #[snafu(display("Failed to retrieve compression options of column {column_name}"))]
1142    CompressionOptions {
1143        #[snafu(implicit)]
1144        location: Location,
1145        source: datatypes::error::Error,
1146        column_name: String,
1147    },
1148
1149    #[snafu(display("Failed to create fulltext index creator"))]
1150    CreateFulltextCreator {
1151        source: index::fulltext_index::error::Error,
1152        #[snafu(implicit)]
1153        location: Location,
1154    },
1155
1156    #[snafu(display("Failed to cast vector of {from} to {to}"))]
1157    CastVector {
1158        #[snafu(implicit)]
1159        location: Location,
1160        from: ConcreteDataType,
1161        to: ConcreteDataType,
1162        source: datatypes::error::Error,
1163    },
1164
1165    #[snafu(display("Failed to push text to fulltext index"))]
1166    FulltextPushText {
1167        source: index::fulltext_index::error::Error,
1168        #[snafu(implicit)]
1169        location: Location,
1170    },
1171
1172    #[snafu(display("Failed to finalize fulltext index creator"))]
1173    FulltextFinish {
1174        source: index::fulltext_index::error::Error,
1175        #[snafu(implicit)]
1176        location: Location,
1177    },
1178
1179    #[snafu(display("Failed to apply fulltext index"))]
1180    ApplyFulltextIndex {
1181        source: index::fulltext_index::error::Error,
1182        #[snafu(implicit)]
1183        location: Location,
1184    },
1185
1186    #[snafu(display("SST file {} does not contain valid stats info", file_path))]
1187    StatsNotPresent {
1188        file_path: String,
1189        #[snafu(implicit)]
1190        location: Location,
1191    },
1192
1193    #[snafu(display("Failed to decode stats of file {}", file_path))]
1194    DecodeStats {
1195        file_path: String,
1196        #[snafu(implicit)]
1197        location: Location,
1198    },
1199
1200    #[snafu(display("Invalid SST primary key range: {reason}"))]
1201    InvalidPrimaryKeyRange {
1202        reason: String,
1203        #[snafu(implicit)]
1204        location: Location,
1205    },
1206
1207    #[snafu(display("Failed to decode SST primary key range {endpoint} endpoint"))]
1208    DecodePrimaryKeyRange {
1209        endpoint: &'static str,
1210        source: mito_codec::error::Error,
1211        #[snafu(implicit)]
1212        location: Location,
1213    },
1214
1215    #[snafu(display("Region {} is busy", region_id))]
1216    RegionBusy {
1217        region_id: RegionId,
1218        #[snafu(implicit)]
1219        location: Location,
1220    },
1221
1222    #[snafu(display("Failed to get schema metadata"))]
1223    GetSchemaMetadata {
1224        source: common_meta::error::Error,
1225        #[snafu(implicit)]
1226        location: Location,
1227    },
1228
1229    #[snafu(display("Timeout"))]
1230    Timeout {
1231        #[snafu(source)]
1232        error: Elapsed,
1233        #[snafu(implicit)]
1234        location: Location,
1235    },
1236
1237    #[snafu(display("Failed to read file metadata"))]
1238    Metadata {
1239        #[snafu(source)]
1240        error: std::io::Error,
1241        #[snafu(implicit)]
1242        location: Location,
1243    },
1244
1245    #[snafu(display("Failed to push value to bloom filter"))]
1246    PushBloomFilterValue {
1247        source: index::bloom_filter::error::Error,
1248        #[snafu(implicit)]
1249        location: Location,
1250    },
1251
1252    #[snafu(display("Failed to finish bloom filter"))]
1253    BloomFilterFinish {
1254        source: index::bloom_filter::error::Error,
1255        #[snafu(implicit)]
1256        location: Location,
1257    },
1258
1259    #[snafu(display("Manual compaction is override by following operations."))]
1260    ManualCompactionOverride {},
1261
1262    #[snafu(display("Manual compaction is already running for region {region_id}."))]
1263    ManualCompactionAlreadyRunning { region_id: RegionId },
1264
1265    #[snafu(display("Compaction is cancelled."))]
1266    CompactionCancelled {},
1267
1268    #[snafu(display("Flush is cancelled."))]
1269    FlushCancelled {},
1270
1271    #[snafu(display("Compaction memory exhausted for region {region_id} (policy: {policy})",))]
1272    CompactionMemoryExhausted {
1273        region_id: RegionId,
1274        policy: String,
1275        #[snafu(source)]
1276        source: common_memory_manager::Error,
1277        #[snafu(implicit)]
1278        location: Location,
1279    },
1280
1281    #[snafu(display(
1282        "Incompatible WAL provider change. This is typically caused by changing WAL provider in database config file without completely cleaning existing files. Global provider: {}, region provider: {}",
1283        global,
1284        region
1285    ))]
1286    IncompatibleWalProviderChange { global: String, region: String },
1287
1288    #[snafu(display("Expected mito manifest info"))]
1289    MitoManifestInfo {
1290        #[snafu(implicit)]
1291        location: Location,
1292    },
1293
1294    #[snafu(display("Failed to scan series"))]
1295    ScanSeries {
1296        #[snafu(implicit)]
1297        location: Location,
1298        source: Arc<Error>,
1299    },
1300
1301    #[snafu(display("Partition {} scan multiple times", partition))]
1302    ScanMultiTimes {
1303        partition: usize,
1304        #[snafu(implicit)]
1305        location: Location,
1306    },
1307
1308    #[snafu(display("Invalid partition expression: {}", expr))]
1309    InvalidPartitionExpr {
1310        expr: String,
1311        #[snafu(implicit)]
1312        location: Location,
1313        source: partition::error::Error,
1314    },
1315
1316    #[snafu(display("Failed to decode bulk wal entry"))]
1317    ConvertBulkWalEntry {
1318        #[snafu(implicit)]
1319        location: Location,
1320        source: common_grpc::Error,
1321    },
1322
1323    #[snafu(display("Failed to encode"))]
1324    Encode {
1325        #[snafu(implicit)]
1326        location: Location,
1327        source: mito_codec::error::Error,
1328    },
1329
1330    #[snafu(display("Failed to decode"))]
1331    Decode {
1332        #[snafu(implicit)]
1333        location: Location,
1334        source: mito_codec::error::Error,
1335    },
1336
1337    #[snafu(display("Unexpected: {reason}"))]
1338    Unexpected {
1339        reason: String,
1340        #[snafu(implicit)]
1341        location: Location,
1342    },
1343
1344    #[cfg(feature = "enterprise")]
1345    #[snafu(display("Failed to scan external range"))]
1346    ScanExternalRange {
1347        source: BoxedError,
1348        #[snafu(implicit)]
1349        location: Location,
1350    },
1351
1352    #[snafu(display(
1353        "Inconsistent timestamp column length, expect: {}, actual: {}",
1354        expected,
1355        actual
1356    ))]
1357    InconsistentTimestampLength {
1358        expected: usize,
1359        actual: usize,
1360        #[snafu(implicit)]
1361        location: Location,
1362    },
1363
1364    #[snafu(display(
1365        "Too many files to read concurrently: {}, max allowed: {}",
1366        actual,
1367        max
1368    ))]
1369    TooManyFilesToRead {
1370        actual: usize,
1371        max: usize,
1372        #[snafu(implicit)]
1373        location: Location,
1374    },
1375
1376    #[snafu(display("Duration out of range: {input:?}"))]
1377    DurationOutOfRange {
1378        input: std::time::Duration,
1379        #[snafu(source)]
1380        error: chrono::OutOfRangeError,
1381        #[snafu(implicit)]
1382        location: Location,
1383    },
1384
1385    #[snafu(display("GC job permit exhausted"))]
1386    TooManyGcJobs {
1387        #[snafu(implicit)]
1388        location: Location,
1389    },
1390
1391    #[snafu(display(
1392        "Staging partition expr mismatch, manifest: {:?}, request: {}",
1393        manifest_expr,
1394        request_expr
1395    ))]
1396    StagingPartitionExprMismatch {
1397        manifest_expr: Option<String>,
1398        request_expr: String,
1399        #[snafu(implicit)]
1400        location: Location,
1401    },
1402
1403    #[snafu(display(
1404        "Invalid source and target region, source: {}, target: {}",
1405        source_region_id,
1406        target_region_id
1407    ))]
1408    InvalidSourceAndTargetRegion {
1409        source_region_id: RegionId,
1410        target_region_id: RegionId,
1411        #[snafu(implicit)]
1412        location: Location,
1413    },
1414
1415    #[snafu(display("Failed to prune file"))]
1416    PruneFile {
1417        source: Arc<Error>,
1418        #[snafu(implicit)]
1419        location: Location,
1420    },
1421
1422    #[snafu(display("Failed to cast column"))]
1423    CastColumn {
1424        #[snafu(source)]
1425        error: datafusion::error::DataFusionError,
1426        #[snafu(implicit)]
1427        location: Location,
1428    },
1429
1430    #[snafu(display("Failed to generate Arrow schema from Parquet file: {}", file))]
1431    ParquetToArrowSchema {
1432        file: String,
1433        #[snafu(source)]
1434        error: parquet::errors::ParquetError,
1435        #[snafu(implicit)]
1436        location: Location,
1437    },
1438
1439    #[snafu(display(
1440        "Region {} is in {:?} state, expect: Writable, Staging or Downgrading",
1441        region_id,
1442        state
1443    ))]
1444    FlushableRegionState {
1445        region_id: RegionId,
1446        state: RegionRoleState,
1447        #[snafu(implicit)]
1448        location: Location,
1449    },
1450}
1451
1452pub type Result<T, E = Error> = std::result::Result<T, E>;
1453
1454impl Error {
1455    /// Returns true if we need to fill default value for a region.
1456    pub(crate) fn is_fill_default(&self) -> bool {
1457        matches!(self, Error::FillDefault { .. })
1458    }
1459
1460    /// Returns true if the file is not found on the object store.
1461    pub(crate) fn is_object_not_found(&self) -> bool {
1462        match self {
1463            Error::OpenDal { error, .. } => error.kind() == ErrorKind::NotFound,
1464            Error::ManifestDeltaNotFound { .. } => true,
1465            Error::LoadSstMeta { source, .. } => source.is_object_not_found(),
1466            _ => false,
1467        }
1468    }
1469
1470    /// Returns whether a failed manifest update may have been persisted.
1471    ///
1472    /// Unknown errors are treated conservatively because deleting output SSTs after an ambiguous
1473    /// manifest write could leave the manifest referencing missing files. The retry layer returns
1474    /// the last attempt's error, so an error that can follow a stored attempt (throttling, or a 403
1475    /// from a revoked key, a changed bucket policy or clock skew) never proves nothing was written.
1476    pub(crate) fn may_have_persisted_manifest_update(&self) -> bool {
1477        match self {
1478            Error::UpdateManifest { .. }
1479            | Error::RegionState { .. }
1480            | Error::RegionTruncated { .. }
1481            | Error::RegionStopped { .. }
1482            | Error::SerdeJson { .. }
1483            | Error::CompressObject { .. } => false,
1484            Error::OpenDal { error, .. } => !matches!(
1485                error.kind(),
1486                ErrorKind::Unsupported
1487                    | ErrorKind::ConfigInvalid
1488                    | ErrorKind::NotFound
1489                    | ErrorKind::IsADirectory
1490                    | ErrorKind::NotADirectory
1491                    | ErrorKind::AlreadyExists
1492                    | ErrorKind::IsSameFile
1493                    | ErrorKind::ConditionNotMatch
1494                    | ErrorKind::RangeNotSatisfied
1495            ),
1496            _ => true,
1497        }
1498    }
1499}
1500
1501impl ErrorExt for Error {
1502    fn status_code(&self) -> StatusCode {
1503        use Error::*;
1504
1505        match self {
1506            DataTypeMismatch { source, .. } => source.status_code(),
1507            OpenDal { .. } | ManifestDeltaNotFound { .. } | ReadParquet { .. } => {
1508                StatusCode::StorageUnavailable
1509            }
1510            WriteWal { source, .. }
1511            | ReadWal { source, .. }
1512            | DeleteWal { source, .. }
1513            | WaitWalDurable { source, .. }
1514            | CheckWalHistory { source, .. } => source.status_code(),
1515            CompressObject { .. }
1516            | DecompressObject { .. }
1517            | SerdeJson { .. }
1518            | Utf8 { .. }
1519            | NewRecordBatch { .. }
1520            | RegionCorrupted { .. }
1521            | InconsistentFile { .. }
1522            | CreateDefault { .. }
1523            | InvalidParquet { .. }
1524            | OperateAbortedIndex { .. }
1525            | IndexEncodeNull { .. }
1526            | NoCheckpoint { .. }
1527            | NoManifests { .. }
1528            | FilesLost { .. }
1529            | InstallManifestTo { .. }
1530            | Unexpected { .. }
1531            | SerializeColumnMetadata { .. }
1532            | SerializeManifest { .. }
1533            | StagingPartitionExprMismatch { .. } => StatusCode::Unexpected,
1534
1535            RegionNotFound { .. } => StatusCode::RegionNotFound,
1536            ObjectStoreNotFound { .. }
1537            | InvalidScanIndex { .. }
1538            | InvalidMeta { .. }
1539            | InvalidRequest { .. }
1540            | PartitionExprVersionMismatch { .. }
1541            | FillDefault { .. }
1542            | ConvertColumnDataType { .. }
1543            | ColumnNotFound { .. }
1544            | InvalidMetadata { .. }
1545            | InvalidRegionOptions { .. }
1546            | InvalidWalReadRequest { .. }
1547            | PartitionOutOfRange { .. }
1548            | ParseJobId { .. }
1549            | DurationOutOfRange { .. }
1550            | MissingOldManifest { .. }
1551            | MissingNewManifest { .. }
1552            | MissingManifest { .. }
1553            | NoOldManifests { .. }
1554            | MissingPartitionExpr { .. }
1555            | SerializePartitionExpr { .. }
1556            | InvalidSourceAndTargetRegion { .. } => StatusCode::InvalidArguments,
1557
1558            IncrementalQueryStale { .. } | SnapshotFenceStale { .. } => StatusCode::RequestOutdated,
1559
1560            SequenceRangeUnsupported { .. } => StatusCode::Unsupported,
1561
1562            RegionSequenceDomainBroken { .. }
1563            | RegionMetadataNotFound { .. }
1564            | Join { .. }
1565            | WorkerStopped { .. }
1566            | Recv { .. }
1567            | DecodeWal { .. }
1568            | ComputeArrow { .. }
1569            | EvalPartitionFilter { .. }
1570            | MergeCandidateSeries { .. }
1571            | BiErrors { .. }
1572            | StopScheduler { .. }
1573            | ComputeVector { .. }
1574            | EncodeMemtable { .. }
1575            | CreateDir { .. }
1576            | ReadDataPart { .. }
1577            | BuildEntry { .. }
1578            | Metadata { .. }
1579            | CastColumn { .. }
1580            | MitoManifestInfo { .. }
1581            | ParquetToArrowSchema { .. } => StatusCode::Internal,
1582
1583            FetchManifests { source, .. } => source.status_code(),
1584
1585            OpenRegion { source, .. } => source.status_code(),
1586
1587            WriteParquet { .. } => StatusCode::StorageUnavailable,
1588            WriteGroup { source, .. } | LoadSstMeta { source, .. } => source.status_code(),
1589            InvalidBatch { .. } => StatusCode::InvalidArguments,
1590            InvalidRecordBatch { .. } => StatusCode::InvalidArguments,
1591            ConvertVector { source, .. } => source.status_code(),
1592
1593            PrimaryKeyLengthMismatch { .. } => StatusCode::InvalidArguments,
1594            InvalidSender { .. } => StatusCode::InvalidArguments,
1595            InvalidSchedulerState { .. } => StatusCode::InvalidArguments,
1596            RegionRequirement { .. } => StatusCode::InvalidArguments,
1597            DeleteSsts { .. } | DeleteIndex { .. } | DeleteIndexes { .. } => {
1598                StatusCode::StorageUnavailable
1599            }
1600            FlushRegion { source, .. } | BuildIndexAsync { source, .. } => source.status_code(),
1601            IndexBuildIncomplete { .. } => StatusCode::IllegalState,
1602            RegionDropped { .. } => StatusCode::Cancelled,
1603            RegionClosed { .. } => StatusCode::Cancelled,
1604            StaleCompactionExecution { .. } => StatusCode::Cancelled,
1605            RegionTruncated { .. } => StatusCode::Cancelled,
1606            RejectWrite { .. } | RejectWriteInStaging { .. } => StatusCode::StorageUnavailable,
1607            CompactRegion { source, .. } => source.status_code(),
1608            EditRegion { source, .. } => source.status_code(),
1609            CompatReader { .. } => StatusCode::Unexpected,
1610            InvalidRegionRequest { source, .. } => source.status_code(),
1611            RegionState { .. } | UpdateManifest { .. } => StatusCode::RegionNotReady,
1612            RegionLeaseExpired { .. } | UnacknowledgedWrite { .. } => StatusCode::RegionNotReady,
1613            JsonOptions { .. } => StatusCode::InvalidArguments,
1614            EmptyRegionDir { .. } | EmptyManifestDir { .. } => StatusCode::RegionNotFound,
1615            ConvertValue { source, .. } => source.status_code(),
1616            ApplyBloomFilterIndex { source, .. } => source.status_code(),
1617            InvalidPartitionExpr { source, .. } => source.status_code(),
1618            BuildIndexApplier { source, .. }
1619            | PushIndexValue { source, .. }
1620            | ApplyInvertedIndex { source, .. }
1621            | IndexFinish { source, .. } => source.status_code(),
1622            PuffinReadBlob { source, .. }
1623            | PuffinAddBlob { source, .. }
1624            | PuffinInitStager { source, .. }
1625            | PuffinBuildReader { source, .. }
1626            | PuffinPurgeStager { source, .. } => source.status_code(),
1627            CleanDir { .. } => StatusCode::Unexpected,
1628            InvalidConfig { .. } => StatusCode::InvalidArguments,
1629            StaleLogEntry { .. }
1630            | InvalidNativeHistogramSubfield { .. }
1631            | InvalidNativeHistogramFieldId { .. } => StatusCode::Unexpected,
1632
1633            External { source, .. } => source.status_code(),
1634
1635            RecordBatch { source, .. } => source.status_code(),
1636
1637            Download { .. } | Upload { .. } => StatusCode::StorageUnavailable,
1638            ChecksumMismatch { .. } => StatusCode::Unexpected,
1639            RegionStopped { .. } => StatusCode::RegionNotReady,
1640            TimeRangePredicateOverflow { .. } => StatusCode::InvalidArguments,
1641            UnsupportedOperation { .. } => StatusCode::Unsupported,
1642            RemoteCompaction { .. } => StatusCode::Unexpected,
1643
1644            IndexOptions { source, .. } => source.status_code(),
1645            CompressionOptions { source, .. } => source.status_code(),
1646            CreateFulltextCreator { source, .. } => source.status_code(),
1647            CastVector { source, .. } => source.status_code(),
1648            FulltextPushText { source, .. }
1649            | FulltextFinish { source, .. }
1650            | ApplyFulltextIndex { source, .. } => source.status_code(),
1651            DecodeStats { .. }
1652            | StatsNotPresent { .. }
1653            | InvalidPrimaryKeyRange { .. }
1654            | DecodePrimaryKeyRange { .. } => StatusCode::Internal,
1655            RegionBusy { .. } => StatusCode::RegionBusy,
1656            GetSchemaMetadata { source, .. } => source.status_code(),
1657            Timeout { .. } => StatusCode::Cancelled,
1658
1659            DecodeArrowRowGroup { .. } => StatusCode::Internal,
1660
1661            PushBloomFilterValue { source, .. } | BloomFilterFinish { source, .. } => {
1662                source.status_code()
1663            }
1664
1665            ManualCompactionOverride {} | CompactionCancelled {} | FlushCancelled {} => {
1666                StatusCode::Cancelled
1667            }
1668
1669            // A concurrent manual compaction fails fast instead of being queued;
1670            // the conflict is reported to the caller, which decides whether to
1671            // issue a new request after the running one finishes.
1672            ManualCompactionAlreadyRunning { .. } => StatusCode::RegionBusy,
1673
1674            CompactionMemoryExhausted { source, .. } => source.status_code(),
1675
1676            IncompatibleWalProviderChange { .. } => StatusCode::InvalidArguments,
1677
1678            ScanSeries { source, .. } => source.status_code(),
1679
1680            ScanMultiTimes { .. } => StatusCode::InvalidArguments,
1681            ConvertBulkWalEntry { source, .. } => source.status_code(),
1682
1683            Encode { source, .. } | Decode { source, .. } => source.status_code(),
1684
1685            #[cfg(feature = "enterprise")]
1686            ScanExternalRange { source, .. } => source.status_code(),
1687
1688            InconsistentTimestampLength { .. } => StatusCode::InvalidArguments,
1689
1690            TooManyFilesToRead { .. } | TooManyGcJobs { .. } => StatusCode::RateLimited,
1691
1692            PruneFile { source, .. } => source.status_code(),
1693
1694            FlushableRegionState { .. } => StatusCode::RegionNotReady,
1695        }
1696    }
1697
1698    fn as_any(&self) -> &dyn Any {
1699        self
1700    }
1701
1702    fn retry_hint(&self) -> RetryHint {
1703        use Error::*;
1704
1705        match self {
1706            PartitionExprVersionMismatch { .. }
1707            | RegionNotFound { .. }
1708            | RegionClosed { .. }
1709            | RejectWriteInStaging { .. }
1710            | RegionLeaseExpired { .. }
1711            | UnacknowledgedWrite { .. }
1712            | RegionState {
1713                state: RegionRoleState::Follower,
1714                ..
1715            } => RetryHint::RetryAfterRouteRefresh,
1716            ReadParquet { .. }
1717            | WriteParquet { .. }
1718            | RejectWrite { .. }
1719            | Download { .. }
1720            | Upload { .. }
1721            | RegionState { .. }
1722            | UpdateManifest { .. }
1723            | RegionStopped { .. }
1724            | RegionBusy { .. }
1725            | ManualCompactionAlreadyRunning { .. }
1726            | FlushableRegionState { .. }
1727            | ManifestDeltaNotFound { .. } => RetryHint::Retryable,
1728
1729            OpenDal { error, .. }
1730            | DeleteSsts { error, .. }
1731            | DeleteIndex { error, .. }
1732            | DeleteIndexes { error, .. } => retry_hint_from_opendal_error(error),
1733
1734            WriteWal { source, .. }
1735            | ReadWal { source, .. }
1736            | DeleteWal { source, .. }
1737            | WaitWalDurable { source, .. }
1738            | CheckWalHistory { source, .. }
1739            | FetchManifests { source, .. }
1740            | External { source, .. } => source.retry_hint(),
1741
1742            OpenRegion { source, .. }
1743            | LoadSstMeta { source, .. }
1744            | WriteGroup { source, .. }
1745            | FlushRegion { source, .. }
1746            | BuildIndexAsync { source, .. }
1747            | CompactRegion { source, .. }
1748            | EditRegion { source, .. }
1749            | ScanSeries { source, .. }
1750            | PruneFile { source, .. } => source.retry_hint(),
1751
1752            DataTypeMismatch { source, .. }
1753            | ConvertVector { source, .. }
1754            | ConvertValue { source, .. }
1755            | IndexOptions { source, .. }
1756            | CompressionOptions { source, .. }
1757            | CastVector { source, .. } => source.retry_hint(),
1758
1759            BuildIndexApplier { source, .. }
1760            | PushIndexValue { source, .. }
1761            | ApplyInvertedIndex { source, .. }
1762            | IndexFinish { source, .. } => source.retry_hint(),
1763
1764            ApplyBloomFilterIndex { source, .. }
1765            | PushBloomFilterValue { source, .. }
1766            | BloomFilterFinish { source, .. } => source.retry_hint(),
1767
1768            PuffinReadBlob { source, .. }
1769            | PuffinAddBlob { source, .. }
1770            | PuffinInitStager { source, .. }
1771            | PuffinBuildReader { source, .. }
1772            | PuffinPurgeStager { source, .. } => source.retry_hint(),
1773
1774            CreateFulltextCreator { source, .. }
1775            | FulltextPushText { source, .. }
1776            | FulltextFinish { source, .. }
1777            | ApplyFulltextIndex { source, .. } => source.retry_hint(),
1778
1779            InvalidRegionRequest { source, .. } => source.retry_hint(),
1780            InvalidPartitionExpr { source, .. } => source.retry_hint(),
1781            RecordBatch { source, .. } => source.retry_hint(),
1782            GetSchemaMetadata { source, .. } => source.retry_hint(),
1783            CompactionMemoryExhausted { source, .. } => source.retry_hint(),
1784            ConvertBulkWalEntry { source, .. } => source.retry_hint(),
1785            Encode { source, .. } | Decode { source, .. } => source.retry_hint(),
1786
1787            #[cfg(feature = "enterprise")]
1788            ScanExternalRange { source, .. } => source.retry_hint(),
1789
1790            _ => RetryHint::NonRetryable,
1791        }
1792    }
1793}
1794
1795#[cfg(test)]
1796mod tests {
1797    use snafu::IntoError;
1798
1799    use super::*;
1800    use crate::region::RegionLeaderState;
1801
1802    #[test]
1803    fn test_partition_version_mismatch_retry_hint() {
1804        let error = PartitionExprVersionMismatchSnafu {
1805            region_id: RegionId::new(1, 1),
1806            request_version: 1u64,
1807            expected_version: 2u64,
1808        }
1809        .build();
1810        assert_eq!(error.status_code(), StatusCode::InvalidArguments);
1811        assert_eq!(error.retry_hint(), RetryHint::RetryAfterRouteRefresh);
1812    }
1813
1814    #[test]
1815    fn test_region_state_retry_hint() {
1816        let region_id = RegionId::new(1, 1);
1817        let writable = RegionRoleState::Leader(RegionLeaderState::Writable);
1818        let follower_error = RegionStateSnafu {
1819            region_id,
1820            state: RegionRoleState::Follower,
1821            expect: writable,
1822        }
1823        .build();
1824        assert_eq!(
1825            follower_error.retry_hint(),
1826            RetryHint::RetryAfterRouteRefresh
1827        );
1828        assert_eq!(follower_error.status_code(), StatusCode::RegionNotReady);
1829        let leader_error = RegionStateSnafu {
1830            region_id,
1831            state: writable,
1832            expect: RegionRoleState::Follower,
1833        }
1834        .build();
1835        assert_eq!(leader_error.retry_hint(), RetryHint::Retryable);
1836        for (error, status, hint) in [
1837            (
1838                RejectWriteSnafu { region_id }.build(),
1839                StatusCode::StorageUnavailable,
1840                RetryHint::Retryable,
1841            ),
1842            (
1843                RejectWriteInStagingSnafu { region_id }.build(),
1844                StatusCode::StorageUnavailable,
1845                RetryHint::RetryAfterRouteRefresh,
1846            ),
1847            (
1848                RegionNotFoundSnafu { region_id }.build(),
1849                StatusCode::RegionNotFound,
1850                RetryHint::RetryAfterRouteRefresh,
1851            ),
1852            (
1853                RegionClosedSnafu { region_id }.build(),
1854                StatusCode::Cancelled,
1855                RetryHint::RetryAfterRouteRefresh,
1856            ),
1857            (
1858                RegionDroppedSnafu { region_id }.build(),
1859                StatusCode::Cancelled,
1860                RetryHint::NonRetryable,
1861            ),
1862            (
1863                RegionTruncatedSnafu { region_id }.build(),
1864                StatusCode::Cancelled,
1865                RetryHint::NonRetryable,
1866            ),
1867        ] {
1868            assert_eq!(error.status_code(), status);
1869            assert_eq!(error.retry_hint(), hint);
1870        }
1871    }
1872
1873    #[test]
1874    fn test_shared_sst_metadata_preserves_error_semantics() {
1875        for source in [
1876            OpenDalSnafu {}
1877                .into_error(object_store::Error::new(ErrorKind::NotFound, "missing SST")),
1878            OpenDalSnafu {}.into_error(object_store::Error::new(
1879                ErrorKind::Unexpected,
1880                "read failed",
1881            )),
1882            RejectWriteSnafu {
1883                region_id: RegionId::new(1, 1),
1884            }
1885            .build(),
1886        ] {
1887            let expected_not_found = source.is_object_not_found();
1888            let expected_status = source.status_code();
1889            let expected_retry = source.retry_hint();
1890            let source = Arc::new(source);
1891            let wrapped = LoadSstMetaSnafu.into_error(source);
1892            assert_eq!(expected_not_found, wrapped.is_object_not_found());
1893            assert_eq!(expected_status, wrapped.status_code());
1894            assert_eq!(expected_retry, wrapped.retry_hint());
1895        }
1896    }
1897
1898    #[test]
1899    fn test_manifest_update_persistence() {
1900        let rejected_kinds = [
1901            ErrorKind::Unsupported,
1902            ErrorKind::ConfigInvalid,
1903            ErrorKind::NotFound,
1904            ErrorKind::IsADirectory,
1905            ErrorKind::NotADirectory,
1906            ErrorKind::AlreadyExists,
1907            ErrorKind::IsSameFile,
1908            ErrorKind::ConditionNotMatch,
1909            ErrorKind::RangeNotSatisfied,
1910        ];
1911        for kind in rejected_kinds {
1912            let error = OpenDalSnafu {}.into_error(object_store::Error::new(kind, "test"));
1913            assert!(
1914                !error.may_have_persisted_manifest_update(),
1915                "error kind {kind:?} should prove the manifest was not persisted"
1916            );
1917        }
1918
1919        // The last retry can fail with these kinds even though an earlier attempt was stored.
1920        for kind in [
1921            ErrorKind::Unexpected,
1922            ErrorKind::RateLimited,
1923            ErrorKind::PermissionDenied,
1924        ] {
1925            let error = OpenDalSnafu {}.into_error(object_store::Error::new(kind, "test"));
1926            assert!(
1927                error.may_have_persisted_manifest_update(),
1928                "error kind {kind:?} should not prove the manifest was not persisted"
1929            );
1930        }
1931
1932        let region_id = RegionId::new(1, 1);
1933        let error = RegionTruncatedSnafu { region_id }.build();
1934        assert!(!error.may_have_persisted_manifest_update());
1935
1936        // This can be raised while applying an edit after its manifest file was saved.
1937        let error = RegionMetadataNotFoundSnafu {}.build();
1938        assert!(error.may_have_persisted_manifest_update());
1939    }
1940}
mito2/error.rs

mito2/
error.rs
