rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-01-04 12:02:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

guard against partial uploads to local storage

Browse Source
This commit is contained in:
Dmitry Rodionov
2022-03-17 21:36:17 +04:00
committed by Dmitry Rodionov
parent 7738254f83
commit b19870cd88
1 changed files with 29 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
pageserver/src/remote_storage/local_fs.rs
View File

@@ -5,6 +5,7 @@
//! volume is mounted to the local FS.
use std::{
ffi::OsString,
future::Future,
path::{Path, PathBuf},
pin::Pin,
@@ -83,11 +84,21 @@ impl RemoteStorage for LocalFs {
) -> anyhow::Result<()> {
let target_file_path = self.resolve_in_storage(to)?;
create_target_directory(&target_file_path).await?;
// We need this dance with sort of durable rename (without fsyncs)
// to prevent partial uploads. This was really hit when pageserver shutdown
// cancelled the upload and partial file was left on the fs
let mut temp_extension = target_file_path
.extension()
.unwrap_or_default()
.to_os_string();
temp_extension.push(OsString::from(".temp"));
let temp_file_path = target_file_path.with_extension(temp_extension);
let mut destination = io::BufWriter::new(
fs::OpenOptions::new()
.write(true)
.create(true)
.open(&target_file_path)
.open(&temp_file_path)
.await
.with_context(|| {
format!(
@@ -101,16 +112,26 @@ impl RemoteStorage for LocalFs {
.await
.with_context(|| {
format!(
"Failed to upload file to the local storage at '{}'",
"Failed to upload file (write temp) to the local storage at '{}'",
temp_file_path.display()
)
})?;
destination.flush().await.with_context(|| {
format!(
"Failed to upload (flush temp) file to the local storage at '{}'",
temp_file_path.display()
)
})?;
fs::rename(temp_file_path, &target_file_path)
.await
.with_context(|| {
format!(
"Failed to upload (rename) file to the local storage at '{}'",
target_file_path.display()
)
})?;
destination.flush().await.with_context(|| {
format!(
"Failed to upload file to the local storage at '{}'",
target_file_path.display()
)
})?;
Ok(())
}