1use std::any::Any;
16use std::sync::Arc;
17use std::time::Duration;
18
19use common_error::define_into_tonic_status;
20use common_error::ext::{BoxedError, ErrorExt, RetryHint};
21use common_error::status_code::StatusCode;
22use common_macro::stack_trace_debug;
23use common_runtime::JoinError;
24use snafu::{Location, Snafu};
25use store_api::storage::RegionId;
26use table::error::Error as TableError;
27use tokio::time::error::Elapsed;
28
29#[derive(Snafu)]
31#[snafu(visibility(pub))]
32#[stack_trace_debug]
33pub enum Error {
34 #[snafu(display("Failed to write a batch group"))]
35 BatchWrite { source: std::sync::Arc<Error> },
36
37 #[snafu(display("Failed to execute async task"))]
38 AsyncTaskExecute {
39 #[snafu(implicit)]
40 location: Location,
41 source: Arc<Error>,
42 },
43
44 #[snafu(display("Failed to watch change"))]
45 WatchAsyncTaskChange {
46 #[snafu(implicit)]
47 location: Location,
48 #[snafu(source)]
49 error: tokio::sync::watch::error::RecvError,
50 },
51
52 #[snafu(display("Failed to handle heartbeat response"))]
53 HandleHeartbeatResponse {
54 #[snafu(implicit)]
55 location: Location,
56 source: common_meta::error::Error,
57 },
58
59 #[snafu(display("Failed to get info from meta server"))]
60 GetMetadata {
61 #[snafu(implicit)]
62 location: Location,
63 source: common_meta::error::Error,
64 },
65
66 #[snafu(display("Failed to execute logical plan"))]
67 ExecuteLogicalPlan {
68 #[snafu(implicit)]
69 location: Location,
70 source: query::error::Error,
71 },
72
73 #[snafu(display("Failed to join datanode runtime task, request_type: {}", request_type))]
74 RuntimeJoin {
75 request_type: &'static str,
76 #[snafu(source)]
77 error: JoinError,
78 #[snafu(implicit)]
79 location: Location,
80 },
81
82 #[snafu(display("Failed to create plan decoder"))]
83 NewPlanDecoder {
84 #[snafu(implicit)]
85 location: Location,
86 source: query::error::Error,
87 },
88
89 #[snafu(display("Failed to decode logical plan"))]
90 DecodeLogicalPlan {
91 #[snafu(implicit)]
92 location: Location,
93 source: common_query::error::Error,
94 },
95
96 #[snafu(display("Schema not found: {}", name))]
97 SchemaNotFound {
98 name: String,
99 #[snafu(implicit)]
100 location: Location,
101 },
102
103 #[snafu(display("Missing timestamp column in request"))]
104 MissingTimestampColumn {
105 #[snafu(implicit)]
106 location: Location,
107 },
108
109 #[snafu(display("Failed to delete value from table: {}", table_name))]
110 Delete {
111 table_name: String,
112 #[snafu(implicit)]
113 location: Location,
114 source: TableError,
115 },
116
117 #[snafu(display("Failed to start server"))]
118 StartServer {
119 #[snafu(implicit)]
120 location: Location,
121 source: servers::error::Error,
122 },
123
124 #[snafu(display("Failed to parse address {}", addr))]
125 ParseAddr {
126 addr: String,
127 #[snafu(source)]
128 error: std::net::AddrParseError,
129 },
130
131 #[snafu(display("Failed to create directory {}", dir))]
132 CreateDir {
133 dir: String,
134 #[snafu(source)]
135 error: std::io::Error,
136 },
137
138 #[snafu(display("Failed to remove directory {}", dir))]
139 RemoveDir {
140 dir: String,
141 #[snafu(source)]
142 error: std::io::Error,
143 },
144
145 #[snafu(display("Failed to open log store"))]
146 OpenLogStore {
147 #[snafu(implicit)]
148 location: Location,
149 source: Box<log_store::error::Error>,
150 },
151
152 #[snafu(display("Invalid query context: {}", reason))]
153 InvalidQueryContext {
154 reason: String,
155 #[snafu(implicit)]
156 location: Location,
157 },
158
159 #[snafu(display("Invalid SQL, error: {}", msg))]
160 InvalidSql { msg: String },
161
162 #[snafu(display("Illegal primary keys definition: {}", msg))]
163 IllegalPrimaryKeysDef {
164 msg: String,
165 #[snafu(implicit)]
166 location: Location,
167 },
168
169 #[snafu(display("Schema {} already exists", name))]
170 SchemaExists {
171 name: String,
172 #[snafu(implicit)]
173 location: Location,
174 },
175
176 #[snafu(display("Failed to initialize meta client"))]
177 MetaClientInit {
178 #[snafu(implicit)]
179 location: Location,
180 source: meta_client::error::Error,
181 },
182
183 #[snafu(display("Missing node id in Datanode config"))]
184 MissingNodeId {
185 #[snafu(implicit)]
186 location: Location,
187 },
188
189 #[snafu(display("Failed to build datanode"))]
190 BuildDatanode {
191 #[snafu(implicit)]
192 location: Location,
193 source: BoxedError,
194 },
195
196 #[snafu(display("Failed to build http client"))]
197 BuildHttpClient {
198 #[snafu(implicit)]
199 location: Location,
200 #[snafu(source)]
201 error: reqwest::Error,
202 },
203
204 #[snafu(display("Missing required field: {}", name))]
205 MissingRequiredField {
206 name: String,
207 #[snafu(implicit)]
208 location: Location,
209 },
210
211 #[snafu(display(
212 "No valid default value can be built automatically, column: {}",
213 column,
214 ))]
215 ColumnNoneDefaultValue {
216 column: String,
217 #[snafu(implicit)]
218 location: Location,
219 },
220
221 #[snafu(display("Failed to shutdown server"))]
222 ShutdownServer {
223 #[snafu(implicit)]
224 location: Location,
225 #[snafu(source)]
226 source: servers::error::Error,
227 },
228
229 #[snafu(display("Failed to shutdown instance"))]
230 ShutdownInstance {
231 #[snafu(implicit)]
232 location: Location,
233 #[snafu(source)]
234 source: BoxedError,
235 },
236
237 #[snafu(display("Payload not exist"))]
238 PayloadNotExist {
239 #[snafu(implicit)]
240 location: Location,
241 },
242
243 #[snafu(display("Unexpected, violated: {}", violated))]
244 Unexpected {
245 violated: String,
246 #[snafu(implicit)]
247 location: Location,
248 },
249
250 #[snafu(display("Failed to handle request for region {}", region_id))]
251 HandleRegionRequest {
252 region_id: RegionId,
253 #[snafu(implicit)]
254 location: Location,
255 source: BoxedError,
256 },
257
258 #[snafu(display("Failed to open batch regions"))]
259 HandleBatchOpenRequest {
260 #[snafu(implicit)]
261 location: Location,
262 source: BoxedError,
263 },
264
265 #[snafu(display("Failed to handle batch ddl request, ddl_type: {}", ddl_type))]
266 HandleBatchDdlRequest {
267 #[snafu(implicit)]
268 location: Location,
269 source: BoxedError,
270 ddl_type: String,
271 },
272
273 #[snafu(display("RegionId {} not found", region_id))]
274 RegionNotFound {
275 region_id: RegionId,
276 #[snafu(implicit)]
277 location: Location,
278 },
279
280 #[snafu(display("Region {} not ready", region_id))]
281 RegionNotReady {
282 region_id: RegionId,
283 #[snafu(implicit)]
284 location: Location,
285 },
286
287 #[snafu(display("Region {} is busy", region_id))]
288 RegionBusy {
289 region_id: RegionId,
290 #[snafu(implicit)]
291 location: Location,
292 },
293
294 #[snafu(display("Region engine {} is not registered", name))]
295 RegionEngineNotFound {
296 name: String,
297 #[snafu(implicit)]
298 location: Location,
299 },
300
301 #[snafu(display(
302 "GC configuration mismatch: metasrv.gc.enable={}, datanode.region_engine.mito.gc.enable={}",
303 metasrv_gc_enabled,
304 datanode_gc_enabled,
305 ))]
306 GcConfigMismatch {
307 metasrv_gc_enabled: bool,
308 datanode_gc_enabled: bool,
309 #[snafu(implicit)]
310 location: Location,
311 },
312
313 #[snafu(display(
314 "Duplicate region engine config '{engine}' at region_engine[{first_index}] and region_engine[{duplicate_index}] (indices are zero-based); each engine type may be configured only once"
315 ))]
316 DuplicateRegionEngineConfig {
317 engine: &'static str,
318 first_index: usize,
319 duplicate_index: usize,
320 #[snafu(implicit)]
321 location: Location,
322 },
323
324 #[snafu(display("Invalid object store WAL config, {} = {:?}: {}", field, value, reason))]
325 InvalidObjectStoreWalConfig {
326 field: &'static str,
327 value: String,
328 reason: String,
329 #[snafu(implicit)]
330 location: Location,
331 },
332
333 #[snafu(display("Object store WAL is only supported in standalone mode"))]
334 ObjectStoreWalNotStandalone {
335 #[snafu(implicit)]
336 location: Location,
337 },
338
339 #[snafu(display("Unsupported output type, expected: {}", expected))]
340 UnsupportedOutput {
341 expected: String,
342 #[snafu(implicit)]
343 location: Location,
344 },
345
346 #[snafu(display("Failed to build region requests"))]
347 BuildRegionRequests {
348 #[snafu(implicit)]
349 location: Location,
350 source: store_api::metadata::MetadataError,
351 },
352
353 #[snafu(display("Failed to serialize WAL options for region {}", region_id))]
354 SerializeWalOptions {
355 region_id: RegionId,
356 #[snafu(source)]
357 error: serde_json::Error,
358 #[snafu(implicit)]
359 location: Location,
360 },
361
362 #[snafu(display("Failed to stop region engine {}", name))]
363 StopRegionEngine {
364 name: String,
365 #[snafu(implicit)]
366 location: Location,
367 source: BoxedError,
368 },
369
370 #[snafu(display(
371 "Failed to find logical regions in physical region {}",
372 physical_region_id
373 ))]
374 FindLogicalRegions {
375 physical_region_id: RegionId,
376 source: metric_engine::error::Error,
377 #[snafu(implicit)]
378 location: Location,
379 },
380
381 #[snafu(display("Failed to build mito engine"))]
382 BuildMitoEngine {
383 source: mito2::error::Error,
384 #[snafu(implicit)]
385 location: Location,
386 },
387
388 #[snafu(display("Failed to build metric engine"))]
389 BuildMetricEngine {
390 source: metric_engine::error::Error,
391 #[snafu(implicit)]
392 location: Location,
393 },
394
395 #[snafu(display("Failed to run gc for region {}", region_id))]
396 GcMitoEngine {
397 region_id: RegionId,
398 source: mito2::error::Error,
399 #[snafu(implicit)]
400 location: Location,
401 },
402
403 #[snafu(display("Failed to list SST entries from storage"))]
404 ListStorageSsts {
405 #[snafu(implicit)]
406 location: Location,
407 source: mito2::error::Error,
408 },
409
410 #[snafu(display("Failed to serialize options to TOML"))]
411 TomlFormat {
412 #[snafu(implicit)]
413 location: Location,
414 #[snafu(source(from(common_config::error::Error, Box::new)))]
415 source: Box<common_config::error::Error>,
416 },
417
418 #[snafu(display(
419 "Failed to get region metadata from engine {} for region_id {}",
420 engine,
421 region_id,
422 ))]
423 GetRegionMetadata {
424 engine: String,
425 region_id: RegionId,
426 #[snafu(implicit)]
427 location: Location,
428 source: BoxedError,
429 },
430
431 #[snafu(display("DataFusion"))]
432 DataFusion {
433 #[snafu(source)]
434 error: datafusion::error::DataFusionError,
435 #[snafu(implicit)]
436 location: Location,
437 },
438
439 #[snafu(display("Failed to acquire permit, source closed"))]
440 ConcurrentQueryLimiterClosed {
441 #[snafu(source)]
442 error: tokio::sync::AcquireError,
443 #[snafu(implicit)]
444 location: Location,
445 },
446
447 #[snafu(display("Failed to acquire permit under timeouts"))]
448 ConcurrentQueryLimiterTimeout {
449 #[snafu(source)]
450 error: Elapsed,
451 #[snafu(implicit)]
452 location: Location,
453 },
454
455 #[snafu(display("Cache not found in registry"))]
456 MissingCache {
457 #[snafu(implicit)]
458 location: Location,
459 },
460
461 #[snafu(display("Failed to serialize json"))]
462 SerializeJson {
463 #[snafu(source)]
464 error: serde_json::Error,
465 #[snafu(implicit)]
466 location: Location,
467 },
468
469 #[snafu(display("Failed object store operation"))]
470 ObjectStore {
471 source: object_store::error::Error,
472 #[snafu(implicit)]
473 location: Location,
474 },
475
476 #[snafu(display("Failed to run object storage probe for provider '{name}' at step '{step}'"))]
477 ObjectStoreProbe {
478 name: String,
479 step: &'static str,
480 #[snafu(source)]
481 error: object_store::Error,
482 #[snafu(implicit)]
483 location: Location,
484 },
485
486 #[snafu(display("Object storage probe for provider '{name}' timed out after {timeout:?}"))]
487 ObjectStoreProbeTimeout {
488 name: String,
489 timeout: Duration,
490 #[snafu(implicit)]
491 location: Location,
492 },
493
494 #[snafu(display("Not yet implemented: {what}"))]
495 NotYetImplemented { what: String },
496}
497
498pub type Result<T> = std::result::Result<T, Error>;
499
500impl ErrorExt for Error {
501 fn status_code(&self) -> StatusCode {
502 use Error::*;
503 match self {
504 Self::BatchWrite { source } => source.status_code(),
505 NewPlanDecoder { source, .. } | ExecuteLogicalPlan { source, .. } => {
506 source.status_code()
507 }
508
509 BuildRegionRequests { source, .. } => source.status_code(),
510 HandleHeartbeatResponse { source, .. } | GetMetadata { source, .. } => {
511 source.status_code()
512 }
513
514 DecodeLogicalPlan { source, .. } => source.status_code(),
515
516 Delete { source, .. } => source.status_code(),
517
518 InvalidQueryContext { .. }
519 | InvalidSql { .. }
520 | IllegalPrimaryKeysDef { .. }
521 | MissingTimestampColumn { .. }
522 | SchemaNotFound { .. }
523 | SchemaExists { .. }
524 | MissingNodeId { .. }
525 | ColumnNoneDefaultValue { .. }
526 | MissingRequiredField { .. }
527 | RegionEngineNotFound { .. }
528 | GcConfigMismatch { .. }
529 | ParseAddr { .. }
530 | TomlFormat { .. }
531 | DuplicateRegionEngineConfig { .. }
532 | InvalidObjectStoreWalConfig { .. }
533 | ObjectStoreWalNotStandalone { .. }
534 | BuildDatanode { .. } => StatusCode::InvalidArguments,
535
536 PayloadNotExist { .. }
537 | Unexpected { .. }
538 | SerializeWalOptions { .. }
539 | WatchAsyncTaskChange { .. }
540 | BuildHttpClient { .. } => StatusCode::Unexpected,
541
542 AsyncTaskExecute { source, .. } => source.status_code(),
543
544 CreateDir { .. }
545 | RemoveDir { .. }
546 | ShutdownInstance { .. }
547 | DataFusion { .. }
548 | RuntimeJoin { .. } => StatusCode::Internal,
549
550 RegionNotFound { .. } => StatusCode::RegionNotFound,
551 RegionNotReady { .. } => StatusCode::RegionNotReady,
552 RegionBusy { .. } => StatusCode::RegionBusy,
553
554 StartServer { source, .. } | ShutdownServer { source, .. } => source.status_code(),
555
556 OpenLogStore { source, .. } => source.status_code(),
557 MetaClientInit { source, .. } => source.status_code(),
558 UnsupportedOutput { .. } | NotYetImplemented { .. } => StatusCode::Unsupported,
559 HandleRegionRequest { source, .. }
560 | GetRegionMetadata { source, .. }
561 | HandleBatchOpenRequest { source, .. }
562 | HandleBatchDdlRequest { source, .. } => source.status_code(),
563 StopRegionEngine { source, .. } => source.status_code(),
564
565 FindLogicalRegions { source, .. } => source.status_code(),
566 BuildMitoEngine { source, .. } | GcMitoEngine { source, .. } => source.status_code(),
567 BuildMetricEngine { source, .. } => source.status_code(),
568 ListStorageSsts { source, .. } => source.status_code(),
569 ConcurrentQueryLimiterClosed { .. } | ConcurrentQueryLimiterTimeout { .. } => {
570 StatusCode::RegionBusy
571 }
572 MissingCache { .. } => StatusCode::Internal,
573 SerializeJson { .. } => StatusCode::Internal,
574
575 ObjectStore { source, .. } => source.status_code(),
576 ObjectStoreProbe { .. } | ObjectStoreProbeTimeout { .. } => {
577 StatusCode::StorageUnavailable
578 }
579 }
580 }
581
582 fn as_any(&self) -> &dyn Any {
583 self
584 }
585
586 fn retry_hint(&self) -> RetryHint {
587 use Error::*;
588
589 match self {
590 Self::BatchWrite { source } => source.retry_hint(),
591 RegionNotReady { .. } | RegionNotFound { .. } => RetryHint::RetryAfterRouteRefresh,
592 RegionBusy { .. }
593 | ConcurrentQueryLimiterClosed { .. }
594 | ConcurrentQueryLimiterTimeout { .. }
595 | ObjectStoreProbeTimeout { .. } => RetryHint::Retryable,
596 NewPlanDecoder { source, .. } | ExecuteLogicalPlan { source, .. } => {
597 source.retry_hint()
598 }
599 HandleHeartbeatResponse { source, .. } | GetMetadata { source, .. } => {
600 source.retry_hint()
601 }
602 DecodeLogicalPlan { source, .. } => source.retry_hint(),
603 Delete { source, .. } => source.retry_hint(),
604 AsyncTaskExecute { source, .. } => source.retry_hint(),
605 StartServer { source, .. } | ShutdownServer { source, .. } => source.retry_hint(),
606 OpenLogStore { source, .. } => source.retry_hint(),
607 MetaClientInit { source, .. } => source.retry_hint(),
608 HandleRegionRequest { source, .. }
609 | GetRegionMetadata { source, .. }
610 | HandleBatchOpenRequest { source, .. }
611 | HandleBatchDdlRequest { source, .. }
612 | StopRegionEngine { source, .. } => source.retry_hint(),
613 FindLogicalRegions { source, .. } => source.retry_hint(),
614 BuildMitoEngine { source, .. } => source.retry_hint(),
615 GcMitoEngine { source, .. } => source.retry_hint(),
616 BuildMetricEngine { source, .. } => source.retry_hint(),
617 ListStorageSsts { source, .. } => source.retry_hint(),
618 ObjectStore { source, .. } => source.retry_hint(),
619 ObjectStoreProbe { error, .. } => {
620 object_store::error::retry_hint_from_opendal_error(error)
621 }
622 _ => RetryHint::NonRetryable,
623 }
624 }
625}
626
627define_into_tonic_status!(Error);
628
629#[cfg(test)]
630mod tests {
631 use common_error::ext::RetryHint;
632
633 use super::*;
634
635 #[test]
636 fn test_region_state_retry_hints() {
637 let region_id = RegionId::new(1024, 1);
638 for (error, status, hint) in [
639 (
640 RegionBusySnafu { region_id }.build(),
641 StatusCode::RegionBusy,
642 RetryHint::Retryable,
643 ),
644 (
645 RegionNotReadySnafu { region_id }.build(),
646 StatusCode::RegionNotReady,
647 RetryHint::RetryAfterRouteRefresh,
648 ),
649 (
650 RegionNotFoundSnafu { region_id }.build(),
651 StatusCode::RegionNotFound,
652 RetryHint::RetryAfterRouteRefresh,
653 ),
654 ] {
655 assert_eq!(error.status_code(), status);
656 assert_eq!(error.retry_hint(), hint);
657 }
658 }
659
660 #[test]
661 fn test_default_hint_is_non_retryable() {
662 let err = UnexpectedSnafu {
663 violated: "mock error",
664 }
665 .build();
666
667 assert_eq!(err.retry_hint(), RetryHint::NonRetryable);
668 }
669}