1use std::any::Any;
16use std::sync::Arc;
17
18use common_error::ext::{ErrorExt, RetryHint, retry_hint_from_io_error};
19use common_error::status_code::StatusCode;
20use common_macro::stack_trace_debug;
21use common_runtime::error::Error as RuntimeError;
22use common_wal::kafka::rskafka_client_error_to_retry_hint;
23use object_store::error::retry_hint_from_opendal_error;
24use serde_json::error::Error as JsonError;
25use snafu::{Location, Snafu};
26use store_api::storage::RegionId;
27
28#[derive(Snafu)]
29#[snafu(visibility(pub))]
30#[stack_trace_debug]
31pub enum Error {
32 #[snafu(display("Failed to create TLS Config"))]
33 TlsConfig {
34 #[snafu(implicit)]
35 location: Location,
36 source: common_wal::error::Error,
37 },
38
39 #[snafu(display("Invalid provider type, expected: {}, actual: {}", expected, actual))]
40 InvalidProvider {
41 #[snafu(implicit)]
42 location: Location,
43 expected: String,
44 actual: String,
45 },
46
47 #[snafu(display("Failed to start log store task: {}", name))]
48 StartWalTask {
49 name: String,
50 #[snafu(implicit)]
51 location: Location,
52 source: RuntimeError,
53 },
54
55 #[snafu(display("Failed to stop log store task: {}", name))]
56 StopWalTask {
57 name: String,
58 #[snafu(implicit)]
59 location: Location,
60 source: RuntimeError,
61 },
62
63 #[snafu(display("Failed to add entry to LogBatch"))]
64 AddEntryLogBatch {
65 #[snafu(source)]
66 error: raft_engine::Error,
67 #[snafu(implicit)]
68 location: Location,
69 },
70
71 #[snafu(display("Failed to perform raft-engine operation"))]
72 RaftEngine {
73 #[snafu(source)]
74 error: raft_engine::Error,
75 #[snafu(implicit)]
76 location: Location,
77 },
78
79 #[snafu(display("Failed to perform IO on path: {}", path))]
80 Io {
81 path: String,
82 #[snafu(source)]
83 error: std::io::Error,
84 #[snafu(implicit)]
85 location: Location,
86 },
87
88 #[snafu(display("Log store not started yet"))]
89 IllegalState {
90 #[snafu(implicit)]
91 location: Location,
92 },
93
94 #[snafu(display("Namespace is illegal: {}", ns))]
95 IllegalNamespace {
96 ns: u64,
97 #[snafu(implicit)]
98 location: Location,
99 },
100
101 #[snafu(display(
102 "Failed to fetch entries from namespace: {}, start: {}, end: {}, max size: {}",
103 ns,
104 start,
105 end,
106 max_size,
107 ))]
108 FetchEntry {
109 ns: u64,
110 start: u64,
111 end: u64,
112 max_size: usize,
113 #[snafu(source)]
114 error: raft_engine::Error,
115 #[snafu(implicit)]
116 location: Location,
117 },
118
119 #[snafu(display(
120 "Cannot override compacted entry, namespace: {}, first index: {}, attempt index: {}",
121 namespace,
122 first_index,
123 attempt_index
124 ))]
125 OverrideCompactedEntry {
126 namespace: u64,
127 first_index: u64,
128 attempt_index: u64,
129 #[snafu(implicit)]
130 location: Location,
131 },
132
133 #[snafu(display(
134 "Failed to build a Kafka client, broker endpoints: {:?}",
135 broker_endpoints
136 ))]
137 BuildClient {
138 broker_endpoints: Vec<String>,
139 #[snafu(implicit)]
140 location: Location,
141 #[snafu(source)]
142 error: rskafka::client::error::Error,
143 },
144
145 #[snafu(display(
146 "Failed to build a Kafka partition client, topic: {}, partition: {}",
147 topic,
148 partition
149 ))]
150 BuildPartitionClient {
151 topic: String,
152 partition: i32,
153 #[snafu(implicit)]
154 location: Location,
155 #[snafu(source)]
156 error: rskafka::client::error::Error,
157 },
158
159 #[snafu(display("Missing required key in a record"))]
160 MissingKey {
161 #[snafu(implicit)]
162 location: Location,
163 },
164
165 #[snafu(display("Missing required value in a record"))]
166 MissingValue {
167 #[snafu(implicit)]
168 location: Location,
169 },
170
171 #[snafu(display("Failed to produce records to Kafka, topic: {}, size: {}", topic, size))]
172 ProduceRecord {
173 topic: String,
174 size: usize,
175 #[snafu(implicit)]
176 location: Location,
177 #[snafu(source)]
178 error: rskafka::client::producer::Error,
179 },
180
181 #[snafu(display("Failed to produce batch records to Kafka"))]
182 BatchProduce {
183 #[snafu(implicit)]
184 location: Location,
185 #[snafu(source)]
186 error: rskafka::client::error::Error,
187 },
188
189 #[snafu(display("Failed to read a record from Kafka, topic: {}", topic))]
190 ConsumeRecord {
191 topic: String,
192 #[snafu(implicit)]
193 location: Location,
194 #[snafu(source)]
195 error: rskafka::client::error::Error,
196 },
197
198 #[snafu(display("Failed to get the latest offset, topic: {}", topic))]
199 GetOffset {
200 topic: String,
201 #[snafu(implicit)]
202 location: Location,
203 #[snafu(source)]
204 error: rskafka::client::error::Error,
205 },
206
207 #[snafu(display("Failed to do a cast"))]
208 Cast {
209 #[snafu(implicit)]
210 location: Location,
211 },
212
213 #[snafu(display("Failed to encode object into json"))]
214 EncodeJson {
215 #[snafu(implicit)]
216 location: Location,
217 #[snafu(source)]
218 error: JsonError,
219 },
220
221 #[snafu(display("Failed to decode object from json"))]
222 DecodeJson {
223 #[snafu(implicit)]
224 location: Location,
225 #[snafu(source)]
226 error: JsonError,
227 },
228
229 #[snafu(display("The record sequence is not legal, error: {}", error))]
230 IllegalSequence {
231 #[snafu(implicit)]
232 location: Location,
233 error: String,
234 },
235
236 #[snafu(display(
237 "Attempt to append discontinuous log entry, region: {}, last index: {}, attempt index: {}",
238 region_id,
239 last_index,
240 attempt_index
241 ))]
242 DiscontinuousLogIndex {
243 region_id: RegionId,
244 last_index: u64,
245 attempt_index: u64,
246 },
247
248 #[snafu(display("OrderedBatchProducer is stopped",))]
249 OrderedBatchProducerStopped {
250 #[snafu(implicit)]
251 location: Location,
252 },
253
254 #[snafu(display("Failed to wait for ProduceResultReceiver"))]
255 WaitProduceResultReceiver {
256 #[snafu(implicit)]
257 location: Location,
258 #[snafu(source)]
259 error: tokio::sync::oneshot::error::RecvError,
260 },
261
262 #[snafu(display(
263 "The length of meta if exceeded the limit: {}, actual: {}",
264 limit,
265 actual
266 ))]
267 MetaLengthExceededLimit {
268 #[snafu(implicit)]
269 location: Location,
270 limit: usize,
271 actual: usize,
272 },
273
274 #[snafu(display("No max value"))]
275 NoMaxValue {
276 #[snafu(implicit)]
277 location: Location,
278 },
279
280 #[snafu(display("Corrupted WAL object, {}", reason))]
281 CorruptedWalObject {
282 reason: String,
283 #[snafu(implicit)]
284 location: Location,
285 },
286
287 #[snafu(display("Invalid WAL object store, {}", reason))]
288 InvalidWalObjectStore {
289 reason: String,
290 #[snafu(implicit)]
291 location: Location,
292 },
293
294 #[snafu(display(
295 "Object store WAL region mismatch, supplied region: {}, {}",
296 region_id,
297 reason
298 ))]
299 MismatchedWalRegion {
300 region_id: RegionId,
301 reason: String,
302 #[snafu(implicit)]
303 location: Location,
304 },
305
306 #[snafu(display(
307 "Invalid WAL entry range, region: {}, start: {}, end: {}",
308 region_id,
309 start_entry_id,
310 end_entry_id
311 ))]
312 InvalidWalEntryRange {
313 region_id: RegionId,
314 start_entry_id: u64,
315 end_entry_id: u64,
316 #[snafu(implicit)]
317 location: Location,
318 },
319
320 #[snafu(display("Incomplete multipart WAL entry of region {}", region_id))]
321 IncompleteWalEntry {
322 region_id: RegionId,
323 #[snafu(implicit)]
324 location: Location,
325 },
326
327 #[snafu(display("WAL object sequence is exhausted, last sequence: {}", last_object_seq))]
328 WalObjectSequenceExhausted {
329 last_object_seq: u64,
330 #[snafu(implicit)]
331 location: Location,
332 },
333
334 #[snafu(display(
335 "WAL object sequence {} is not settled: the open batch has assigned entry ids under it",
336 object_seq
337 ))]
338 WalObjectSequenceUnsettled {
339 object_seq: u64,
340 #[snafu(implicit)]
341 location: Location,
342 },
343
344 #[snafu(display(
345 "WAL entry positions of region {} in one object are exhausted",
346 region_id
347 ))]
348 WalEntryPositionExhausted {
349 region_id: RegionId,
350 #[snafu(implicit)]
351 location: Location,
352 },
353
354 #[snafu(display("WAL object already exists with different content, path: {}", path))]
355 WalObjectConflict {
356 path: String,
357 #[snafu(implicit)]
358 location: Location,
359 },
360
361 #[snafu(display(
362 "WAL object {} was written by the earlier epoch {}, this store writes epoch {}",
363 path,
364 existing_epoch,
365 epoch
366 ))]
367 StaleWalObject {
368 path: String,
369 existing_epoch: u64,
370 epoch: u64,
371 #[snafu(implicit)]
372 location: Location,
373 },
374
375 #[snafu(display(
376 "WAL object {} that starts epoch {} was already present, so the open cannot tell whether it wrote it",
377 path,
378 epoch
379 ))]
380 UnconfirmedWalEpochStart {
381 path: String,
382 epoch: u64,
383 #[snafu(implicit)]
384 location: Location,
385 },
386
387 #[snafu(display("Failed to {} WAL object, path: {}", operation, path))]
388 WalObjectStore {
389 operation: &'static str,
390 path: String,
391 #[snafu(source)]
392 error: object_store::Error,
393 #[snafu(implicit)]
394 location: Location,
395 },
396
397 #[snafu(display("Failed to fetch a segment of WAL object, path: {}", path))]
398 WalSegmentFetch {
399 path: String,
400 #[snafu(source)]
401 error: tokio::task::JoinError,
402 #[snafu(implicit)]
403 location: Location,
404 },
405
406 #[snafu(display("Invalid WAL object, path: {}", path))]
407 InvalidWalObject {
408 path: String,
409 #[snafu(source(from(Error, Box::new)))]
410 source: Box<Error>,
411 #[snafu(implicit)]
412 location: Location,
413 },
414
415 #[snafu(display(
416 "Object store WAL prefix mismatch, expected: {}, actual: {}",
417 expected,
418 actual
419 ))]
420 MismatchedWalPrefix {
421 expected: String,
422 actual: String,
423 #[snafu(implicit)]
424 location: Location,
425 },
426
427 #[snafu(display("Object store WAL log store is stopped"))]
428 ObjectStoreWalStopped {
429 #[snafu(implicit)]
430 location: Location,
431 },
432
433 #[snafu(display("Object store WAL operation failed"))]
434 ObjectStoreWal {
435 source: Arc<Error>,
436 #[snafu(implicit)]
437 location: Location,
438 },
439
440 #[snafu(display(
441 "Object store WAL rejects appends until the object store accepts its acknowledged backlog"
442 ))]
443 ObjectStoreWalFailing {
444 source: Arc<Error>,
445 #[snafu(implicit)]
446 location: Location,
447 },
448}
449
450pub type Result<T> = std::result::Result<T, Error>;
451
452fn rskafka_client_error_to_status_code(error: &rskafka::client::error::Error) -> StatusCode {
453 match error {
454 rskafka::client::error::Error::Connection(_)
455 | rskafka::client::error::Error::Request(_)
456 | rskafka::client::error::Error::InvalidResponse(_)
457 | rskafka::client::error::Error::ServerError { .. }
458 | rskafka::client::error::Error::RetryFailed(_) => StatusCode::Internal,
459 rskafka::client::error::Error::Timeout => StatusCode::StorageUnavailable,
460 _ => StatusCode::Internal,
461 }
462}
463
464impl ErrorExt for Error {
465 fn as_any(&self) -> &dyn Any {
466 self
467 }
468
469 fn status_code(&self) -> StatusCode {
470 use Error::*;
471
472 match self {
473 TlsConfig { .. }
474 | InvalidProvider { .. }
475 | IllegalNamespace { .. }
476 | MissingKey { .. }
477 | MissingValue { .. }
478 | OverrideCompactedEntry { .. }
479 | InvalidWalObjectStore { .. }
480 | MismatchedWalPrefix { .. }
481 | MismatchedWalRegion { .. }
482 | IncompleteWalEntry { .. }
483 | InvalidWalEntryRange { .. } => StatusCode::InvalidArguments,
484 StartWalTask { .. }
485 | StopWalTask { .. }
486 | IllegalState { .. }
487 | NoMaxValue { .. }
488 | Cast { .. }
489 | EncodeJson { .. }
490 | DecodeJson { .. }
491 | IllegalSequence { .. }
492 | DiscontinuousLogIndex { .. }
493 | OrderedBatchProducerStopped { .. }
494 | WaitProduceResultReceiver { .. }
495 | MetaLengthExceededLimit { .. }
496 | ObjectStoreWalStopped { .. }
497 | WalSegmentFetch { .. } => StatusCode::Internal,
498
499 CorruptedWalObject { .. }
500 | WalObjectConflict { .. }
501 | WalObjectSequenceExhausted { .. }
502 | WalEntryPositionExhausted { .. } => StatusCode::Unexpected,
503 WalObjectSequenceUnsettled { .. } => StatusCode::IllegalState,
504
505 InvalidWalObject { source, .. } => source.status_code(),
506 ObjectStoreWal { source, .. } => source.status_code(),
507 ObjectStoreWalFailing { .. } => StatusCode::StorageUnavailable,
508
509 WalObjectStore { .. }
511 | StaleWalObject { .. }
512 | UnconfirmedWalEpochStart { .. }
513 | Io { .. } => StatusCode::StorageUnavailable,
514 FetchEntry { .. } | RaftEngine { .. } | AddEntryLogBatch { .. } => {
516 StatusCode::StorageUnavailable
517 }
518 ProduceRecord { error, .. } => match error {
520 rskafka::client::producer::Error::Client(error) => {
521 rskafka_client_error_to_status_code(error)
522 }
523 rskafka::client::producer::Error::Aggregator(_)
524 | rskafka::client::producer::Error::FlushError(_)
525 | rskafka::client::producer::Error::TooLarge => StatusCode::Internal,
526 },
527 BuildClient { error, .. }
528 | BuildPartitionClient { error, .. }
529 | BatchProduce { error, .. }
530 | GetOffset { error, .. }
531 | ConsumeRecord { error, .. } => rskafka_client_error_to_status_code(error),
532 }
533 }
534
535 fn retry_hint(&self) -> RetryHint {
536 use Error::*;
537
538 match self {
539 WalObjectStore { error, .. } => retry_hint_from_opendal_error(error),
540 ObjectStoreWal { source, .. } => source.retry_hint(),
541 Io { error, .. } => retry_hint_from_io_error(error),
542 FetchEntry { .. }
543 | RaftEngine { .. }
544 | AddEntryLogBatch { .. }
545 | WalObjectSequenceUnsettled { .. }
546 | StaleWalObject { .. }
547 | UnconfirmedWalEpochStart { .. }
548 | ObjectStoreWalFailing { .. } => RetryHint::Retryable,
549 ProduceRecord { error, .. } => match error {
550 rskafka::client::producer::Error::Client(error) => {
551 rskafka_client_error_to_retry_hint(error)
552 }
553 rskafka::client::producer::Error::Aggregator(_)
554 | rskafka::client::producer::Error::FlushError(_)
555 | rskafka::client::producer::Error::TooLarge => RetryHint::NonRetryable,
556 },
557 BuildClient { error, .. }
558 | BuildPartitionClient { error, .. }
559 | BatchProduce { error, .. }
560 | GetOffset { error, .. }
561 | ConsumeRecord { error, .. } => rskafka_client_error_to_retry_hint(error),
562 _ => RetryHint::NonRetryable,
563 }
564 }
565}