diff --git a/pageserver/src/layered_repository.rs b/pageserver/src/layered_repository.rs index 704f04c93d..a245cbd35d 100644 --- a/pageserver/src/layered_repository.rs +++ b/pageserver/src/layered_repository.rs @@ -2358,3 +2358,157 @@ fn rename_to_backup(path: PathBuf) -> anyhow::Result<()> { bail!("couldn't find an unused backup number for {:?}", path) } + +/// +/// Tests that are specific to the layered storage format. +/// +/// There are more unit tests in repository.rs that work through the +/// Repository interface and are expected to work regardless of the +/// file format and directory layout. The test here are more low level. +/// +#[cfg(test)] +mod tests { + use super::*; + use crate::repository::repo_harness::*; + + #[test] + fn corrupt_metadata() -> Result<()> { + const TEST_NAME: &str = "corrupt_metadata"; + let harness = RepoHarness::create(TEST_NAME)?; + let repo = harness.load(); + + repo.create_empty_timeline(TIMELINE_ID, Lsn(0))?; + drop(repo); + + let metadata_path = harness.timeline_path(&TIMELINE_ID).join(METADATA_FILE_NAME); + + assert!(metadata_path.is_file()); + + let mut metadata_bytes = std::fs::read(&metadata_path)?; + assert_eq!(metadata_bytes.len(), 512); + metadata_bytes[512 - 4 - 2] ^= 1; + std::fs::write(metadata_path, metadata_bytes)?; + + let new_repo = harness.load(); + let err = new_repo.get_timeline(TIMELINE_ID).err().unwrap(); + assert_eq!(err.to_string(), "failed to load metadata"); + assert_eq!( + err.source().unwrap().to_string(), + "metadata checksum mismatch" + ); + + Ok(()) + } + + /// + /// Test the logic in 'load_layer_map' that removes layer files that are + /// newer than 'disk_consistent_lsn'. + /// + #[test] + fn future_layerfiles() -> Result<()> { + const TEST_NAME: &str = "future_layerfiles"; + let harness = RepoHarness::create(TEST_NAME)?; + let repo = harness.load(); + + // Create a timeline with disk_consistent_lsn = 8000 + let tline = repo.create_empty_timeline(TIMELINE_ID, Lsn(0x8000))?; + let writer = tline.writer(); + writer.advance_last_record_lsn(Lsn(0x8000)); + drop(writer); + repo.checkpoint_iteration(CheckpointConfig::Forced)?; + drop(repo); + + let timeline_path = harness.timeline_path(&TIMELINE_ID); + + let make_empty_file = |filename: &str| -> std::io::Result<()> { + let path = timeline_path.join(filename); + + assert!(!path.exists()); + std::fs::write(&path, &[])?; + + Ok(()) + }; + + // Helper function to check that a relation file exists, and a corresponding + // .0.old file does not. + let assert_exists = |filename: &str| { + let path = timeline_path.join(filename); + assert!(path.exists(), "file {} was removed", filename); + + // Check that there is no .old file + let backup_path = timeline_path.join(format!("{}.0.old", filename)); + assert!( + !backup_path.exists(), + "unexpected backup file {}", + backup_path.display() + ); + }; + + // Helper function to check that a relation file does *not* exists, and a corresponding + // ..old file does. + let assert_is_renamed = |filename: &str, num: u32| { + let path = timeline_path.join(filename); + assert!( + !path.exists(), + "file {} was not removed as expected", + filename + ); + + let backup_path = timeline_path.join(format!("{}.{}.old", filename, num)); + assert!( + backup_path.exists(), + "backup file {} was not created", + backup_path.display() + ); + }; + + // These files are considered to be in the future and will be renamed out + // of the way + let future_filenames = vec![ + format!("pg_control_0_{:016X}", 0x8001), + format!("pg_control_0_{:016X}_{:016X}", 0x8001, 0x8008), + ]; + // But these are not: + let past_filenames = vec![ + format!("pg_control_0_{:016X}", 0x8000), + format!("pg_control_0_{:016X}_{:016X}", 0x7000, 0x8001), + ]; + + for filename in future_filenames.iter().chain(past_filenames.iter()) { + make_empty_file(filename)?; + } + + // Load the timeline. This will cause the files in the "future" to be renamed + // away. + let new_repo = harness.load(); + new_repo.get_timeline(TIMELINE_ID).unwrap(); + drop(new_repo); + + for filename in future_filenames.iter() { + assert_is_renamed(filename, 0); + } + for filename in past_filenames.iter() { + assert_exists(filename); + } + + // Create the future files again, and load again. They should be renamed to + // *.1.old this time. + for filename in future_filenames.iter() { + make_empty_file(filename)?; + } + + let new_repo = harness.load(); + new_repo.get_timeline(TIMELINE_ID).unwrap(); + drop(new_repo); + + for filename in future_filenames.iter() { + assert_is_renamed(filename, 0); + assert_is_renamed(filename, 1); + } + for filename in past_filenames.iter() { + assert_exists(filename); + } + + Ok(()) + } +} diff --git a/pageserver/src/repository.rs b/pageserver/src/repository.rs index 8527430357..6142953a58 100644 --- a/pageserver/src/repository.rs +++ b/pageserver/src/repository.rs @@ -447,8 +447,6 @@ pub mod repo_harness { #[allow(clippy::bool_assert_comparison)] #[cfg(test)] mod tests { - use crate::layered_repository::metadata::METADATA_FILE_NAME; - use super::repo_harness::*; use super::*; use postgres_ffi::{pg_constants, xlog_utils::SIZEOF_CHECKPOINT}; @@ -1132,141 +1130,4 @@ mod tests { Ok(()) } - - #[test] - fn corrupt_metadata() -> Result<()> { - const TEST_NAME: &str = "corrupt_metadata"; - let harness = RepoHarness::create(TEST_NAME)?; - let repo = harness.load(); - - repo.create_empty_timeline(TIMELINE_ID, Lsn(0))?; - drop(repo); - - let metadata_path = harness.timeline_path(&TIMELINE_ID).join(METADATA_FILE_NAME); - - assert!(metadata_path.is_file()); - - let mut metadata_bytes = std::fs::read(&metadata_path)?; - assert_eq!(metadata_bytes.len(), 512); - metadata_bytes[512 - 4 - 2] ^= 1; - std::fs::write(metadata_path, metadata_bytes)?; - - let new_repo = harness.load(); - let err = new_repo.get_timeline(TIMELINE_ID).err().unwrap(); - assert_eq!(err.to_string(), "failed to load metadata"); - assert_eq!( - err.source().unwrap().to_string(), - "metadata checksum mismatch" - ); - - Ok(()) - } - - #[test] - fn future_layerfiles() -> Result<()> { - const TEST_NAME: &str = "future_layerfiles"; - let harness = RepoHarness::create(TEST_NAME)?; - let repo = harness.load(); - - // Create a timeline with disk_consistent_lsn = 8000 - let tline = repo.create_empty_timeline(TIMELINE_ID, Lsn(0x8000))?; - let writer = tline.writer(); - writer.advance_last_record_lsn(Lsn(0x8000)); - drop(writer); - repo.checkpoint_iteration(CheckpointConfig::Forced)?; - drop(repo); - - let timeline_path = harness.timeline_path(&TIMELINE_ID); - - let make_empty_file = |filename: &str| -> std::io::Result<()> { - let path = timeline_path.join(filename); - - assert!(!path.exists()); - std::fs::write(&path, &[])?; - - Ok(()) - }; - - // Helper function to check that a relation file exists, and a corresponding - // .0.old file does not. - let assert_exists = |filename: &str| { - let path = timeline_path.join(filename); - assert!(path.exists(), "file {} was removed", filename); - - // Check that there is no .old file - let backup_path = timeline_path.join(format!("{}.0.old", filename)); - assert!( - !backup_path.exists(), - "unexpected backup file {}", - backup_path.display() - ); - }; - - // Helper function to check that a relation file does *not* exists, and a corresponding - // ..old file does. - let assert_is_renamed = |filename: &str, num: u32| { - let path = timeline_path.join(filename); - assert!( - !path.exists(), - "file {} was not removed as expected", - filename - ); - - let backup_path = timeline_path.join(format!("{}.{}.old", filename, num)); - assert!( - backup_path.exists(), - "backup file {} was not created", - backup_path.display() - ); - }; - - // These files are considered to be in the future and will be renamed out - // of the way - let future_filenames = vec![ - format!("pg_control_0_{:016X}", 0x8001), - format!("pg_control_0_{:016X}_{:016X}", 0x8001, 0x8008), - ]; - // But these are not: - let past_filenames = vec![ - format!("pg_control_0_{:016X}", 0x8000), - format!("pg_control_0_{:016X}_{:016X}", 0x7000, 0x8001), - ]; - - for filename in future_filenames.iter().chain(past_filenames.iter()) { - make_empty_file(filename)?; - } - - // Load the timeline. This will cause the files in the "future" to be renamed - // away. - let new_repo = harness.load(); - new_repo.get_timeline(TIMELINE_ID).unwrap(); - drop(new_repo); - - for filename in future_filenames.iter() { - assert_is_renamed(filename, 0); - } - for filename in past_filenames.iter() { - assert_exists(filename); - } - - // Create the future files again, and load again. They should be renamed to - // *.1.old this time. - for filename in future_filenames.iter() { - make_empty_file(filename)?; - } - - let new_repo = harness.load(); - new_repo.get_timeline(TIMELINE_ID).unwrap(); - drop(new_repo); - - for filename in future_filenames.iter() { - assert_is_renamed(filename, 0); - assert_is_renamed(filename, 1); - } - for filename in past_filenames.iter() { - assert_exists(filename); - } - - Ok(()) - } }