1use std::any::Any;
16use std::sync::Arc;
17
18use common_error::define_into_tonic_status;
19use common_error::ext::{BoxedError, ErrorExt, RetryHint};
20use common_error::status_code::StatusCode;
21use common_macro::stack_trace_debug;
22use common_runtime::JoinError;
23use snafu::{Location, Snafu};
24use store_api::storage::RegionId;
25use table::error::Error as TableError;
26use tokio::time::error::Elapsed;
27
28#[derive(Snafu)]
30#[snafu(visibility(pub))]
31#[stack_trace_debug]
32pub enum Error {
33 #[snafu(display("Failed to execute async task"))]
34 AsyncTaskExecute {
35 #[snafu(implicit)]
36 location: Location,
37 source: Arc<Error>,
38 },
39
40 #[snafu(display("Failed to watch change"))]
41 WatchAsyncTaskChange {
42 #[snafu(implicit)]
43 location: Location,
44 #[snafu(source)]
45 error: tokio::sync::watch::error::RecvError,
46 },
47
48 #[snafu(display("Failed to handle heartbeat response"))]
49 HandleHeartbeatResponse {
50 #[snafu(implicit)]
51 location: Location,
52 source: common_meta::error::Error,
53 },
54
55 #[snafu(display("Failed to get info from meta server"))]
56 GetMetadata {
57 #[snafu(implicit)]
58 location: Location,
59 source: common_meta::error::Error,
60 },
61
62 #[snafu(display("Failed to execute logical plan"))]
63 ExecuteLogicalPlan {
64 #[snafu(implicit)]
65 location: Location,
66 source: query::error::Error,
67 },
68
69 #[snafu(display("Failed to join datanode runtime task, request_type: {}", request_type))]
70 RuntimeJoin {
71 request_type: &'static str,
72 #[snafu(source)]
73 error: JoinError,
74 #[snafu(implicit)]
75 location: Location,
76 },
77
78 #[snafu(display("Failed to create plan decoder"))]
79 NewPlanDecoder {
80 #[snafu(implicit)]
81 location: Location,
82 source: query::error::Error,
83 },
84
85 #[snafu(display("Failed to decode logical plan"))]
86 DecodeLogicalPlan {
87 #[snafu(implicit)]
88 location: Location,
89 source: common_query::error::Error,
90 },
91
92 #[snafu(display("Schema not found: {}", name))]
93 SchemaNotFound {
94 name: String,
95 #[snafu(implicit)]
96 location: Location,
97 },
98
99 #[snafu(display("Missing timestamp column in request"))]
100 MissingTimestampColumn {
101 #[snafu(implicit)]
102 location: Location,
103 },
104
105 #[snafu(display("Failed to delete value from table: {}", table_name))]
106 Delete {
107 table_name: String,
108 #[snafu(implicit)]
109 location: Location,
110 source: TableError,
111 },
112
113 #[snafu(display("Failed to start server"))]
114 StartServer {
115 #[snafu(implicit)]
116 location: Location,
117 source: servers::error::Error,
118 },
119
120 #[snafu(display("Failed to parse address {}", addr))]
121 ParseAddr {
122 addr: String,
123 #[snafu(source)]
124 error: std::net::AddrParseError,
125 },
126
127 #[snafu(display("Failed to create directory {}", dir))]
128 CreateDir {
129 dir: String,
130 #[snafu(source)]
131 error: std::io::Error,
132 },
133
134 #[snafu(display("Failed to remove directory {}", dir))]
135 RemoveDir {
136 dir: String,
137 #[snafu(source)]
138 error: std::io::Error,
139 },
140
141 #[snafu(display("Failed to open log store"))]
142 OpenLogStore {
143 #[snafu(implicit)]
144 location: Location,
145 source: Box<log_store::error::Error>,
146 },
147
148 #[snafu(display("Invalid query context: {}", reason))]
149 InvalidQueryContext {
150 reason: String,
151 #[snafu(implicit)]
152 location: Location,
153 },
154
155 #[snafu(display("Invalid SQL, error: {}", msg))]
156 InvalidSql { msg: String },
157
158 #[snafu(display("Illegal primary keys definition: {}", msg))]
159 IllegalPrimaryKeysDef {
160 msg: String,
161 #[snafu(implicit)]
162 location: Location,
163 },
164
165 #[snafu(display("Schema {} already exists", name))]
166 SchemaExists {
167 name: String,
168 #[snafu(implicit)]
169 location: Location,
170 },
171
172 #[snafu(display("Failed to initialize meta client"))]
173 MetaClientInit {
174 #[snafu(implicit)]
175 location: Location,
176 source: meta_client::error::Error,
177 },
178
179 #[snafu(display("Missing node id in Datanode config"))]
180 MissingNodeId {
181 #[snafu(implicit)]
182 location: Location,
183 },
184
185 #[snafu(display("Failed to build datanode"))]
186 BuildDatanode {
187 #[snafu(implicit)]
188 location: Location,
189 source: BoxedError,
190 },
191
192 #[snafu(display("Failed to build http client"))]
193 BuildHttpClient {
194 #[snafu(implicit)]
195 location: Location,
196 #[snafu(source)]
197 error: reqwest::Error,
198 },
199
200 #[snafu(display("Missing required field: {}", name))]
201 MissingRequiredField {
202 name: String,
203 #[snafu(implicit)]
204 location: Location,
205 },
206
207 #[snafu(display(
208 "No valid default value can be built automatically, column: {}",
209 column,
210 ))]
211 ColumnNoneDefaultValue {
212 column: String,
213 #[snafu(implicit)]
214 location: Location,
215 },
216
217 #[snafu(display("Failed to shutdown server"))]
218 ShutdownServer {
219 #[snafu(implicit)]
220 location: Location,
221 #[snafu(source)]
222 source: servers::error::Error,
223 },
224
225 #[snafu(display("Failed to shutdown instance"))]
226 ShutdownInstance {
227 #[snafu(implicit)]
228 location: Location,
229 #[snafu(source)]
230 source: BoxedError,
231 },
232
233 #[snafu(display("Payload not exist"))]
234 PayloadNotExist {
235 #[snafu(implicit)]
236 location: Location,
237 },
238
239 #[snafu(display("Unexpected, violated: {}", violated))]
240 Unexpected {
241 violated: String,
242 #[snafu(implicit)]
243 location: Location,
244 },
245
246 #[snafu(display("Failed to handle request for region {}", region_id))]
247 HandleRegionRequest {
248 region_id: RegionId,
249 #[snafu(implicit)]
250 location: Location,
251 source: BoxedError,
252 },
253
254 #[snafu(display("Failed to open batch regions"))]
255 HandleBatchOpenRequest {
256 #[snafu(implicit)]
257 location: Location,
258 source: BoxedError,
259 },
260
261 #[snafu(display("Failed to handle batch ddl request, ddl_type: {}", ddl_type))]
262 HandleBatchDdlRequest {
263 #[snafu(implicit)]
264 location: Location,
265 source: BoxedError,
266 ddl_type: String,
267 },
268
269 #[snafu(display("RegionId {} not found", region_id))]
270 RegionNotFound {
271 region_id: RegionId,
272 #[snafu(implicit)]
273 location: Location,
274 },
275
276 #[snafu(display("Region {} not ready", region_id))]
277 RegionNotReady {
278 region_id: RegionId,
279 #[snafu(implicit)]
280 location: Location,
281 },
282
283 #[snafu(display("Region {} is busy", region_id))]
284 RegionBusy {
285 region_id: RegionId,
286 #[snafu(implicit)]
287 location: Location,
288 },
289
290 #[snafu(display("Region engine {} is not registered", name))]
291 RegionEngineNotFound {
292 name: String,
293 #[snafu(implicit)]
294 location: Location,
295 },
296
297 #[snafu(display(
298 "GC configuration mismatch: metasrv.gc.enable={}, datanode.region_engine.mito.gc.enable={}",
299 metasrv_gc_enabled,
300 datanode_gc_enabled,
301 ))]
302 GcConfigMismatch {
303 metasrv_gc_enabled: bool,
304 datanode_gc_enabled: bool,
305 #[snafu(implicit)]
306 location: Location,
307 },
308
309 #[snafu(display(
310 "Duplicate region engine config '{engine}' at region_engine[{first_index}] and region_engine[{duplicate_index}] (indices are zero-based); each engine type may be configured only once"
311 ))]
312 DuplicateRegionEngineConfig {
313 engine: &'static str,
314 first_index: usize,
315 duplicate_index: usize,
316 #[snafu(implicit)]
317 location: Location,
318 },
319
320 #[snafu(display("Invalid object store WAL config, {} = {:?}: {}", field, value, reason))]
321 InvalidObjectStoreWalConfig {
322 field: &'static str,
323 value: String,
324 reason: String,
325 #[snafu(implicit)]
326 location: Location,
327 },
328
329 #[snafu(display("Object store WAL is only supported in standalone mode"))]
330 ObjectStoreWalNotStandalone {
331 #[snafu(implicit)]
332 location: Location,
333 },
334
335 #[snafu(display("Unsupported output type, expected: {}", expected))]
336 UnsupportedOutput {
337 expected: String,
338 #[snafu(implicit)]
339 location: Location,
340 },
341
342 #[snafu(display("Failed to build region requests"))]
343 BuildRegionRequests {
344 #[snafu(implicit)]
345 location: Location,
346 source: store_api::metadata::MetadataError,
347 },
348
349 #[snafu(display("Failed to serialize WAL options for region {}", region_id))]
350 SerializeWalOptions {
351 region_id: RegionId,
352 #[snafu(source)]
353 error: serde_json::Error,
354 #[snafu(implicit)]
355 location: Location,
356 },
357
358 #[snafu(display("Failed to stop region engine {}", name))]
359 StopRegionEngine {
360 name: String,
361 #[snafu(implicit)]
362 location: Location,
363 source: BoxedError,
364 },
365
366 #[snafu(display(
367 "Failed to find logical regions in physical region {}",
368 physical_region_id
369 ))]
370 FindLogicalRegions {
371 physical_region_id: RegionId,
372 source: metric_engine::error::Error,
373 #[snafu(implicit)]
374 location: Location,
375 },
376
377 #[snafu(display("Failed to build mito engine"))]
378 BuildMitoEngine {
379 source: mito2::error::Error,
380 #[snafu(implicit)]
381 location: Location,
382 },
383
384 #[snafu(display("Failed to build metric engine"))]
385 BuildMetricEngine {
386 source: metric_engine::error::Error,
387 #[snafu(implicit)]
388 location: Location,
389 },
390
391 #[snafu(display("Failed to run gc for region {}", region_id))]
392 GcMitoEngine {
393 region_id: RegionId,
394 source: mito2::error::Error,
395 #[snafu(implicit)]
396 location: Location,
397 },
398
399 #[snafu(display("Failed to list SST entries from storage"))]
400 ListStorageSsts {
401 #[snafu(implicit)]
402 location: Location,
403 source: mito2::error::Error,
404 },
405
406 #[snafu(display("Failed to serialize options to TOML"))]
407 TomlFormat {
408 #[snafu(implicit)]
409 location: Location,
410 #[snafu(source(from(common_config::error::Error, Box::new)))]
411 source: Box<common_config::error::Error>,
412 },
413
414 #[snafu(display(
415 "Failed to get region metadata from engine {} for region_id {}",
416 engine,
417 region_id,
418 ))]
419 GetRegionMetadata {
420 engine: String,
421 region_id: RegionId,
422 #[snafu(implicit)]
423 location: Location,
424 source: BoxedError,
425 },
426
427 #[snafu(display("DataFusion"))]
428 DataFusion {
429 #[snafu(source)]
430 error: datafusion::error::DataFusionError,
431 #[snafu(implicit)]
432 location: Location,
433 },
434
435 #[snafu(display("Failed to acquire permit, source closed"))]
436 ConcurrentQueryLimiterClosed {
437 #[snafu(source)]
438 error: tokio::sync::AcquireError,
439 #[snafu(implicit)]
440 location: Location,
441 },
442
443 #[snafu(display("Failed to acquire permit under timeouts"))]
444 ConcurrentQueryLimiterTimeout {
445 #[snafu(source)]
446 error: Elapsed,
447 #[snafu(implicit)]
448 location: Location,
449 },
450
451 #[snafu(display("Cache not found in registry"))]
452 MissingCache {
453 #[snafu(implicit)]
454 location: Location,
455 },
456
457 #[snafu(display("Failed to serialize json"))]
458 SerializeJson {
459 #[snafu(source)]
460 error: serde_json::Error,
461 #[snafu(implicit)]
462 location: Location,
463 },
464
465 #[snafu(display("Failed object store operation"))]
466 ObjectStore {
467 source: object_store::error::Error,
468 #[snafu(implicit)]
469 location: Location,
470 },
471
472 #[snafu(display("Not yet implemented: {what}"))]
473 NotYetImplemented { what: String },
474}
475
476pub type Result<T> = std::result::Result<T, Error>;
477
478impl ErrorExt for Error {
479 fn status_code(&self) -> StatusCode {
480 use Error::*;
481 match self {
482 NewPlanDecoder { source, .. } | ExecuteLogicalPlan { source, .. } => {
483 source.status_code()
484 }
485
486 BuildRegionRequests { source, .. } => source.status_code(),
487 HandleHeartbeatResponse { source, .. } | GetMetadata { source, .. } => {
488 source.status_code()
489 }
490
491 DecodeLogicalPlan { source, .. } => source.status_code(),
492
493 Delete { source, .. } => source.status_code(),
494
495 InvalidQueryContext { .. }
496 | InvalidSql { .. }
497 | IllegalPrimaryKeysDef { .. }
498 | MissingTimestampColumn { .. }
499 | SchemaNotFound { .. }
500 | SchemaExists { .. }
501 | MissingNodeId { .. }
502 | ColumnNoneDefaultValue { .. }
503 | MissingRequiredField { .. }
504 | RegionEngineNotFound { .. }
505 | GcConfigMismatch { .. }
506 | ParseAddr { .. }
507 | TomlFormat { .. }
508 | DuplicateRegionEngineConfig { .. }
509 | InvalidObjectStoreWalConfig { .. }
510 | ObjectStoreWalNotStandalone { .. }
511 | BuildDatanode { .. } => StatusCode::InvalidArguments,
512
513 PayloadNotExist { .. }
514 | Unexpected { .. }
515 | SerializeWalOptions { .. }
516 | WatchAsyncTaskChange { .. }
517 | BuildHttpClient { .. } => StatusCode::Unexpected,
518
519 AsyncTaskExecute { source, .. } => source.status_code(),
520
521 CreateDir { .. }
522 | RemoveDir { .. }
523 | ShutdownInstance { .. }
524 | DataFusion { .. }
525 | RuntimeJoin { .. } => StatusCode::Internal,
526
527 RegionNotFound { .. } => StatusCode::RegionNotFound,
528 RegionNotReady { .. } => StatusCode::RegionNotReady,
529 RegionBusy { .. } => StatusCode::RegionBusy,
530
531 StartServer { source, .. } | ShutdownServer { source, .. } => source.status_code(),
532
533 OpenLogStore { source, .. } => source.status_code(),
534 MetaClientInit { source, .. } => source.status_code(),
535 UnsupportedOutput { .. } | NotYetImplemented { .. } => StatusCode::Unsupported,
536 HandleRegionRequest { source, .. }
537 | GetRegionMetadata { source, .. }
538 | HandleBatchOpenRequest { source, .. }
539 | HandleBatchDdlRequest { source, .. } => source.status_code(),
540 StopRegionEngine { source, .. } => source.status_code(),
541
542 FindLogicalRegions { source, .. } => source.status_code(),
543 BuildMitoEngine { source, .. } | GcMitoEngine { source, .. } => source.status_code(),
544 BuildMetricEngine { source, .. } => source.status_code(),
545 ListStorageSsts { source, .. } => source.status_code(),
546 ConcurrentQueryLimiterClosed { .. } | ConcurrentQueryLimiterTimeout { .. } => {
547 StatusCode::RegionBusy
548 }
549 MissingCache { .. } => StatusCode::Internal,
550 SerializeJson { .. } => StatusCode::Internal,
551
552 ObjectStore { source, .. } => source.status_code(),
553 }
554 }
555
556 fn as_any(&self) -> &dyn Any {
557 self
558 }
559
560 fn retry_hint(&self) -> RetryHint {
561 use Error::*;
562
563 match self {
564 RegionNotReady { .. } | RegionNotFound { .. } => RetryHint::RetryAfterRouteRefresh,
565 RegionBusy { .. }
566 | ConcurrentQueryLimiterClosed { .. }
567 | ConcurrentQueryLimiterTimeout { .. } => RetryHint::Retryable,
568 NewPlanDecoder { source, .. } | ExecuteLogicalPlan { source, .. } => {
569 source.retry_hint()
570 }
571 HandleHeartbeatResponse { source, .. } | GetMetadata { source, .. } => {
572 source.retry_hint()
573 }
574 DecodeLogicalPlan { source, .. } => source.retry_hint(),
575 Delete { source, .. } => source.retry_hint(),
576 AsyncTaskExecute { source, .. } => source.retry_hint(),
577 StartServer { source, .. } | ShutdownServer { source, .. } => source.retry_hint(),
578 OpenLogStore { source, .. } => source.retry_hint(),
579 MetaClientInit { source, .. } => source.retry_hint(),
580 HandleRegionRequest { source, .. }
581 | GetRegionMetadata { source, .. }
582 | HandleBatchOpenRequest { source, .. }
583 | HandleBatchDdlRequest { source, .. }
584 | StopRegionEngine { source, .. } => source.retry_hint(),
585 FindLogicalRegions { source, .. } => source.retry_hint(),
586 BuildMitoEngine { source, .. } => source.retry_hint(),
587 GcMitoEngine { source, .. } => source.retry_hint(),
588 BuildMetricEngine { source, .. } => source.retry_hint(),
589 ListStorageSsts { source, .. } => source.retry_hint(),
590 ObjectStore { source, .. } => source.retry_hint(),
591 _ => RetryHint::NonRetryable,
592 }
593 }
594}
595
596define_into_tonic_status!(Error);
597
598#[cfg(test)]
599mod tests {
600 use common_error::ext::RetryHint;
601
602 use super::*;
603
604 #[test]
605 fn test_region_state_retry_hints() {
606 let region_id = RegionId::new(1024, 1);
607 for (error, status, hint) in [
608 (
609 RegionBusySnafu { region_id }.build(),
610 StatusCode::RegionBusy,
611 RetryHint::Retryable,
612 ),
613 (
614 RegionNotReadySnafu { region_id }.build(),
615 StatusCode::RegionNotReady,
616 RetryHint::RetryAfterRouteRefresh,
617 ),
618 (
619 RegionNotFoundSnafu { region_id }.build(),
620 StatusCode::RegionNotFound,
621 RetryHint::RetryAfterRouteRefresh,
622 ),
623 ] {
624 assert_eq!(error.status_code(), status);
625 assert_eq!(error.retry_hint(), hint);
626 }
627 }
628
629 #[test]
630 fn test_default_hint_is_non_retryable() {
631 let err = UnexpectedSnafu {
632 violated: "mock error",
633 }
634 .build();
635
636 assert_eq!(err.retry_hint(), RetryHint::NonRetryable);
637 }
638}