diff --git a/Cargo.lock b/Cargo.lock index c1580ba5a1..859fc8af78 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1951,8 +1951,10 @@ dependencies = [ "anyhow", "aversion", "bookfile 0.2.0", + "rand", "serde", "tempfile", + "zenith_utils", ] [[package]] diff --git a/snapfile/Cargo.toml b/snapfile/Cargo.toml index e4cd4d9847..24a1d71c99 100644 --- a/snapfile/Cargo.toml +++ b/snapfile/Cargo.toml @@ -10,6 +10,8 @@ anyhow = "1.0" aversion = "0.2" bookfile = "^0.2" serde = { version = "1.0", features = ["derive"] } +rand = "0.8.3" +zenith_utils = { path = "../zenith_utils" } [dev-dependencies] tempfile = "3.2" diff --git a/snapfile/src/lib.rs b/snapfile/src/lib.rs index 4b63e3cbb3..76ed9fde1f 100644 --- a/snapfile/src/lib.rs +++ b/snapfile/src/lib.rs @@ -13,10 +13,33 @@ use anyhow::{anyhow, bail, Result}; use aversion::group::{DataSink, DataSourceExt}; use aversion::util::cbor::CborData; use bookfile::{Book, BookWriter, ChapterIndex, ChapterWriter}; +use std::ffi::OsString; use std::fs::File; use std::io::Write; -use std::path::Path; -use versioned::PageIndex; +use std::path::{Path, PathBuf}; +use versioned::{PageIndex, Predecessor, SnapFileMeta}; +use zenith_utils::lsn::Lsn; + +impl SnapFileMeta { + fn new(previous: Option, lsn: u64) -> Self { + // Store the metadata of the predecessor snapshot, if there is one. + let predecessor = previous.map(|prev| Predecessor { + id: prev.snap_id, + lsn: prev.lsn, + }); + + let snap_id: u64 = rand::random(); + SnapFileMeta { + snap_id, + predecessor, + lsn, + } + } + + fn to_filename(&self) -> OsString { + format!("{:x}.zdb", self.snap_id).into() + } +} impl PageIndex { /// Retrieve the page offset from the index. @@ -67,6 +90,18 @@ impl SnapFile { }) } + /// Read the snapshot metadata. + pub fn read_meta(&mut self) -> Result { + let chapter_num = self + .book + .find_chapter(versioned::CHAPTER_SNAP_META) + .ok_or_else(|| anyhow!("snapfile missing meta"))?; + let chapter_reader = self.book.chapter_reader(chapter_num)?; + let mut source = CborData::new(chapter_reader); + let meta: SnapFileMeta = source.expect_message()?; + Ok(meta) + } + /// Return the number of pages stored in this snapshot. pub fn page_count(&self) -> usize { self.page_index.page_count() @@ -109,18 +144,36 @@ impl SnapFile { pub struct SnapWriter { writer: ChapterWriter, page_index: PageIndex, + meta: SnapFileMeta, current_offset: u64, } impl SnapWriter { - /// Create a new `SnapWriter` - pub fn new(path: &Path) -> Result { + /// Create a new `SnapWriter`. + /// + /// The LSN is the last page update present in this snapshot. + /// + /// If this is an incremental snapshot, supply the metadata of the previous + /// snapshot. + pub fn new(dir: &Path, previous: Option, lsn: Lsn) -> Result { + let meta = SnapFileMeta::new(previous, lsn.into()); + let mut path = PathBuf::from(dir); + path.push(meta.to_filename()); let file = File::create(path)?; let book = BookWriter::new(file, versioned::SNAPFILE_MAGIC)?; + + // Write a chapter for the snapshot metadata. + let writer = book.new_chapter(versioned::CHAPTER_SNAP_META); + let mut sink = CborData::new(writer); + sink.write_message(&meta)?; + let book = sink.into_inner().close()?; + + // Open a new chapter for raw page data. let writer = book.new_chapter(versioned::CHAPTER_PAGES); Ok(SnapWriter { writer, page_index: PageIndex::default(), + meta, current_offset: 0, }) } @@ -144,7 +197,7 @@ impl SnapWriter { /// /// This consumes the PagesWriter and completes the snapshot. // - pub fn finish(self) -> Result<()> { + pub fn finish(self) -> Result { let book = self.writer.close()?; // Write out a page index and close the book. This will write out any @@ -157,7 +210,9 @@ impl SnapWriter { // Close the chapter, then close the book. sink.into_inner().close()?.close()?; - Ok(()) + + // Return the snapshot metadata to the caller. + Ok(self.meta) } } @@ -171,23 +226,26 @@ mod tests { fn snap_two_pages() { // When `dir` goes out of scope the directory will be unlinked. let dir = TempDir::new().unwrap(); - let mut path = PathBuf::from(dir.path()); - path.push("test.snap"); - { + let snap_meta = { // Write out a new snapshot file with two pages. - let mut snap = SnapWriter::new(&path).unwrap(); + let mut snap = SnapWriter::new(dir.path(), None, Lsn(1234)).unwrap(); // Write the pages out of order, because why not? let page99 = [99u8; 8192]; snap.write_page(99, page99).unwrap(); let page33 = [33u8; 8192]; snap.write_page(33, page33).unwrap(); - snap.finish().unwrap(); - } + snap.finish().unwrap() + }; + + assert_eq!(snap_meta.lsn, 1234); { // Read the snapshot file and verify the contents. + let mut path = PathBuf::from(dir.path()); + path.push(snap_meta.to_filename()); let mut snap = SnapFile::new(&path).unwrap(); + assert_eq!(snap.page_count(), 2); assert!(!snap.has_page(0)); assert!(snap.has_page(33)); assert!(!snap.has_page(98)); @@ -197,6 +255,10 @@ mod tests { assert_eq!(*page.0, [33u8; 8192]); let page = snap.read_page(99).unwrap().unwrap(); assert_eq!(*page.0, [99u8; 8192]); + + // Make sure the deserialized metadata matches what we think we wrote. + let meta2 = snap.read_meta().unwrap(); + assert_eq!(snap_meta, meta2); } } @@ -204,16 +266,16 @@ mod tests { fn snap_zero_pages() { // When `dir` goes out of scope the directory will be unlinked. let dir = TempDir::new().unwrap(); - let mut path = PathBuf::from(dir.path()); - path.push("test.snap"); - { + let snap_meta = { // Write out a new snapshot file with no pages. - let snap = SnapWriter::new(&path).unwrap(); - snap.finish().unwrap(); - } + let snap = SnapWriter::new(dir.path(), None, Lsn(1234)).unwrap(); + snap.finish().unwrap() + }; { // Read the snapshot file. + let mut path = PathBuf::from(dir.path()); + path.push(snap_meta.to_filename()); let mut snap = SnapFile::new(&path).unwrap(); assert_eq!(snap.page_index.page_count(), 0); assert!(!snap.has_page(0)); diff --git a/snapfile/src/versioned.rs b/snapfile/src/versioned.rs index 91a7f9ce73..8105f257b9 100644 --- a/snapfile/src/versioned.rs +++ b/snapfile/src/versioned.rs @@ -18,14 +18,44 @@ pub(crate) const SNAPFILE_MAGIC: u32 = 0x7fb8_38a8; // Constant chapter numbers // FIXME: the bookfile crate should use something better to index, e.g. strings. /// Snapshot-specific file metadata -#[allow(dead_code)] // FIXME: this is a placeholder for future functionality. pub(crate) const CHAPTER_SNAP_META: u64 = 1; /// A packed set of 8KB pages. pub(crate) const CHAPTER_PAGES: u64 = 2; /// An index of pages. pub(crate) const CHAPTER_PAGE_INDEX: u64 = 3; -// FIXME: move serialized data structs to a separate file. +/// Information about the predecessor snapshot. +/// +/// It contains the snap_id of the predecessor snapshot, and the LSN +/// of that snapshot. +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub struct Predecessor { + /// This is the id number of the previous snapshot. + /// + /// This must match the snap_id of the previous snapshot. + pub id: u64, + + /// This is the LSN of the previous snapshot. + pub lsn: u64, +} + +#[derive(Debug, PartialEq, Serialize, Deserialize, Versioned, UpgradeLatest)] +pub struct SnapFileMetaV1 { + /// This is a unique ID number for this snapshot. + /// + /// This number guarantees that snapshot history is unique. + pub snap_id: u64, + + /// Information about the predecessor snapshot. + /// + /// If `None`, this snapshot is the start of a new database. + pub predecessor: Option, + + /// This is the last LSN stored in this snapshot. + pub lsn: u64, +} + +pub type SnapFileMeta = SnapFileMetaV1; /// An index from page number to offset within the pages chapter. #[derive(Debug, Default, Serialize, Deserialize, Versioned, UpgradeLatest)] @@ -40,4 +70,5 @@ pub type PageIndex = PageIndexV1; // Each message gets a unique message id, for tracking by the aversion traits. assign_message_ids! { PageIndex: 100, + SnapFileMeta: 101, }