mirror of
https://github.com/neondatabase/neon.git
synced 2026-01-15 01:12:56 +00:00
75d583c04a
## Problem
During timeline creation we create special mark file which presense
indicates that initialization didnt complete successfully. In case of a
crash restart we can remove such half-initialized timeline and following
retry from control plane side should perform another attempt.
So in case of a possible crash restart during initial loading we have
following picture:
```
timelines
| - <timeline_id>___uninit
| - <timeline_id>
| - | <timeline files>
```
We call `std::fs::read_dir` to walk files in `timelines` directory one
by one. If we see uninit file
we proceed with deletion of both, timeline directory and uninit file. If
we see timeline we check if uninit file exists and do the same cleanup.
But in fact its possible to get both branches to be true at the same
time. Result of readdir doesnt reflect following directory state
modifications. So you can still get "valid" entry on the next iteration
of the loop despite the fact that it was deleted in one of the previous
iterations of the loop.
To see that you can apply the following patch (it disables uninit mark
cleanup on successful timeline creation):
```diff
diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs
index 4beb2664..b3cdad8f 100644
--- a/pageserver/src/tenant.rs
+++ b/pageserver/src/tenant.rs
@@ -224,11 +224,6 @@ impl UninitializedTimeline<'_> {
)
})?;
}
- uninit_mark.remove_uninit_mark().with_context(|| {
- format!(
- "Failed to remove uninit mark file for timeline {tenant_id}/{timeline_id}"
- )
- })?;
v.insert(Arc::clone(&new_timeline));
new_timeline.maybe_spawn_flush_loop();
```
And perform the following steps:
```bash
neon_local init
neon_local start
neon_local tenant create
neon_local stop
neon_local start
```
The error is:
```log
INFO load{tenant_id=X}:blocking: Found an uninit mark file .neon/tenants/X/timelines/Y.___uninit, removing the timeline and its uninit mark
2023-06-09T18:43:41.664247Z ERROR load{tenant_id=X}: load failed, setting tenant state to Broken: failed to load metadata
Caused by:
0: Failed to read metadata bytes from path .neon/tenants/X/timelines/Y/metadata
1: No such file or directory (os error 2)
```
So uninit mark got deleted together with timeline directory but we still
got directory entry for it and tried to load it.
The bug prevented tenant from being successfully loaded.
## Summary of changes
Ideally I think we shouldnt place uninit marks in the same directory as timeline directories but move them to separate directory and
gather them as an input to actual listing, but that would be sort of an
on-disk format change, so just check whether entries are still valid
before operating on them.
112 lines
3.6 KiB
Rust
112 lines
3.6 KiB
Rust
use hyper::{header, Body, Response, StatusCode};
|
|
use serde::{Deserialize, Serialize};
|
|
use std::error::Error as StdError;
|
|
use thiserror::Error;
|
|
use tracing::error;
|
|
|
|
#[derive(Debug, Error)]
|
|
pub enum ApiError {
|
|
#[error("Bad request: {0:#?}")]
|
|
BadRequest(anyhow::Error),
|
|
|
|
#[error("Forbidden: {0}")]
|
|
Forbidden(String),
|
|
|
|
#[error("Unauthorized: {0}")]
|
|
Unauthorized(String),
|
|
|
|
#[error("NotFound: {0}")]
|
|
NotFound(Box<dyn StdError + Send + Sync + 'static>),
|
|
|
|
#[error("Conflict: {0}")]
|
|
Conflict(String),
|
|
|
|
#[error("Precondition failed: {0}")]
|
|
PreconditionFailed(Box<str>),
|
|
|
|
#[error(transparent)]
|
|
InternalServerError(anyhow::Error),
|
|
}
|
|
|
|
impl ApiError {
|
|
pub fn into_response(self) -> Response<Body> {
|
|
match self {
|
|
ApiError::BadRequest(err) => HttpErrorBody::response_from_msg_and_status(
|
|
format!("{err:#?}"), // use debug printing so that we give the cause
|
|
StatusCode::BAD_REQUEST,
|
|
),
|
|
ApiError::Forbidden(_) => {
|
|
HttpErrorBody::response_from_msg_and_status(self.to_string(), StatusCode::FORBIDDEN)
|
|
}
|
|
ApiError::Unauthorized(_) => HttpErrorBody::response_from_msg_and_status(
|
|
self.to_string(),
|
|
StatusCode::UNAUTHORIZED,
|
|
),
|
|
ApiError::NotFound(_) => {
|
|
HttpErrorBody::response_from_msg_and_status(self.to_string(), StatusCode::NOT_FOUND)
|
|
}
|
|
ApiError::Conflict(_) => {
|
|
HttpErrorBody::response_from_msg_and_status(self.to_string(), StatusCode::CONFLICT)
|
|
}
|
|
ApiError::PreconditionFailed(_) => HttpErrorBody::response_from_msg_and_status(
|
|
self.to_string(),
|
|
StatusCode::PRECONDITION_FAILED,
|
|
),
|
|
ApiError::InternalServerError(err) => HttpErrorBody::response_from_msg_and_status(
|
|
err.to_string(),
|
|
StatusCode::INTERNAL_SERVER_ERROR,
|
|
),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Serialize, Deserialize)]
|
|
pub struct HttpErrorBody {
|
|
pub msg: String,
|
|
}
|
|
|
|
impl HttpErrorBody {
|
|
pub fn from_msg(msg: String) -> Self {
|
|
HttpErrorBody { msg }
|
|
}
|
|
|
|
pub fn response_from_msg_and_status(msg: String, status: StatusCode) -> Response<Body> {
|
|
HttpErrorBody { msg }.to_response(status)
|
|
}
|
|
|
|
pub fn to_response(&self, status: StatusCode) -> Response<Body> {
|
|
Response::builder()
|
|
.status(status)
|
|
.header(header::CONTENT_TYPE, "application/json")
|
|
// we do not have nested maps with non string keys so serialization shouldn't fail
|
|
.body(Body::from(serde_json::to_string(self).unwrap()))
|
|
.unwrap()
|
|
}
|
|
}
|
|
|
|
pub async fn route_error_handler(err: routerify::RouteError) -> Response<Body> {
|
|
match err.downcast::<ApiError>() {
|
|
Ok(api_error) => api_error_handler(*api_error),
|
|
Err(other_error) => {
|
|
// We expect all the request handlers to return an ApiError, so this should
|
|
// not be reached. But just in case.
|
|
error!("Error processing HTTP request: {other_error:?}");
|
|
HttpErrorBody::response_from_msg_and_status(
|
|
other_error.to_string(),
|
|
StatusCode::INTERNAL_SERVER_ERROR,
|
|
)
|
|
}
|
|
}
|
|
}
|
|
|
|
pub fn api_error_handler(api_error: ApiError) -> Response<Body> {
|
|
// Print a stack trace for Internal Server errors
|
|
if let ApiError::InternalServerError(_) = api_error {
|
|
error!("Error processing HTTP request: {api_error:?}");
|
|
} else {
|
|
error!("Error processing HTTP request: {api_error:#}");
|
|
}
|
|
|
|
api_error.into_response()
|
|
}
|