snapfile: add snapshot metadata

Add some basic metadata to the snapshot file, including an id number,
predecessor snapshot, and lsn.

Cargo.lock

@@ -1951,8 +1951,10 @@ dependencies = [
  "anyhow",
  "aversion",
  "bookfile 0.2.0",
+ "rand",
  "serde",
  "tempfile",
+ "zenith_utils",
 ]
 
 [[package]]

snapfile/Cargo.toml

@@ -10,6 +10,8 @@ anyhow = "1.0"
 aversion = "0.2"
 bookfile = "^0.2"
 serde = { version = "1.0", features = ["derive"] }
+rand = "0.8.3"
+zenith_utils = { path = "../zenith_utils" }
 
 [dev-dependencies]
 tempfile = "3.2"

snapfile/src/lib.rs

@@ -13,10 +13,33 @@ use anyhow::{anyhow, bail, Result};
 use aversion::group::{DataSink, DataSourceExt};
 use aversion::util::cbor::CborData;
 use bookfile::{Book, BookWriter, ChapterIndex, ChapterWriter};
+use std::ffi::OsString;
 use std::fs::File;
 use std::io::Write;
-use std::path::Path;
-use versioned::PageIndex;
+use std::path::{Path, PathBuf};
+use versioned::{PageIndex, Predecessor, SnapFileMeta};
+use zenith_utils::lsn::Lsn;
+
+impl SnapFileMeta {
+    fn new(previous: Option<SnapFileMeta>, lsn: u64) -> Self {
+        // Store the metadata of the predecessor snapshot, if there is one.
+        let predecessor = previous.map(|prev| Predecessor {
+            id: prev.snap_id,
+            lsn: prev.lsn,
+        });
+
+        let snap_id: u64 = rand::random();
+        SnapFileMeta {
+            snap_id,
+            predecessor,
+            lsn,
+        }
+    }
+
+    fn to_filename(&self) -> OsString {
+        format!("{:x}.zdb", self.snap_id).into()
+    }
+}
 
 impl PageIndex {
     /// Retrieve the page offset from the index.
@@ -67,6 +90,18 @@ impl SnapFile {
         })
     }
 
+    /// Read the snapshot metadata.
+    pub fn read_meta(&mut self) -> Result<SnapFileMeta> {
+        let chapter_num = self
+            .book
+            .find_chapter(versioned::CHAPTER_SNAP_META)
+            .ok_or_else(|| anyhow!("snapfile missing meta"))?;
+        let chapter_reader = self.book.chapter_reader(chapter_num)?;
+        let mut source = CborData::new(chapter_reader);
+        let meta: SnapFileMeta = source.expect_message()?;
+        Ok(meta)
+    }
+
     /// Return the number of pages stored in this snapshot.
     pub fn page_count(&self) -> usize {
         self.page_index.page_count()
@@ -109,18 +144,36 @@ impl SnapFile {
 pub struct SnapWriter {
     writer: ChapterWriter<File>,
     page_index: PageIndex,
+    meta: SnapFileMeta,
     current_offset: u64,
 }
 
 impl SnapWriter {
-    /// Create a new `SnapWriter`
-    pub fn new(path: &Path) -> Result<Self> {
+    /// Create a new `SnapWriter`.
+    ///
+    /// The LSN is the last page update present in this snapshot.
+    ///
+    /// If this is an incremental snapshot, supply the metadata of the previous
+    /// snapshot.
+    pub fn new(dir: &Path, previous: Option<SnapFileMeta>, lsn: Lsn) -> Result<Self> {
+        let meta = SnapFileMeta::new(previous, lsn.into());
+        let mut path = PathBuf::from(dir);
+        path.push(meta.to_filename());
         let file = File::create(path)?;
         let book = BookWriter::new(file, versioned::SNAPFILE_MAGIC)?;
+
+        // Write a chapter for the snapshot metadata.
+        let writer = book.new_chapter(versioned::CHAPTER_SNAP_META);
+        let mut sink = CborData::new(writer);
+        sink.write_message(&meta)?;
+        let book = sink.into_inner().close()?;
+
+        // Open a new chapter for raw page data.
         let writer = book.new_chapter(versioned::CHAPTER_PAGES);
         Ok(SnapWriter {
             writer,
             page_index: PageIndex::default(),
+            meta,
             current_offset: 0,
         })
     }
@@ -144,7 +197,7 @@ impl SnapWriter {
     ///
     /// This consumes the PagesWriter and completes the snapshot.
     //
-    pub fn finish(self) -> Result<()> {
+    pub fn finish(self) -> Result<SnapFileMeta> {
         let book = self.writer.close()?;
 
         // Write out a page index and close the book. This will write out any
@@ -157,7 +210,9 @@ impl SnapWriter {
 
         // Close the chapter, then close the book.
         sink.into_inner().close()?.close()?;
-        Ok(())
+
+        // Return the snapshot metadata to the caller.
+        Ok(self.meta)
     }
 }
 
@@ -171,23 +226,26 @@ mod tests {
     fn snap_two_pages() {
         // When `dir` goes out of scope the directory will be unlinked.
         let dir = TempDir::new().unwrap();
-        let mut path = PathBuf::from(dir.path());
-        path.push("test.snap");
-        {
+        let snap_meta = {
             // Write out a new snapshot file with two pages.
-            let mut snap = SnapWriter::new(&path).unwrap();
+            let mut snap = SnapWriter::new(dir.path(), None, Lsn(1234)).unwrap();
             // Write the pages out of order, because why not?
             let page99 = [99u8; 8192];
             snap.write_page(99, page99).unwrap();
             let page33 = [33u8; 8192];
             snap.write_page(33, page33).unwrap();
-            snap.finish().unwrap();
-        }
+            snap.finish().unwrap()
+        };
+
+        assert_eq!(snap_meta.lsn, 1234);
 
         {
             // Read the snapshot file and verify the contents.
+            let mut path = PathBuf::from(dir.path());
+            path.push(snap_meta.to_filename());
             let mut snap = SnapFile::new(&path).unwrap();
 
+            assert_eq!(snap.page_count(), 2);
             assert!(!snap.has_page(0));
             assert!(snap.has_page(33));
             assert!(!snap.has_page(98));
@@ -197,6 +255,10 @@ mod tests {
             assert_eq!(*page.0, [33u8; 8192]);
             let page = snap.read_page(99).unwrap().unwrap();
             assert_eq!(*page.0, [99u8; 8192]);
+
+            // Make sure the deserialized metadata matches what we think we wrote.
+            let meta2 = snap.read_meta().unwrap();
+            assert_eq!(snap_meta, meta2);
         }
     }
 
@@ -204,16 +266,16 @@ mod tests {
     fn snap_zero_pages() {
         // When `dir` goes out of scope the directory will be unlinked.
         let dir = TempDir::new().unwrap();
-        let mut path = PathBuf::from(dir.path());
-        path.push("test.snap");
-        {
+        let snap_meta = {
             // Write out a new snapshot file with no pages.
-            let snap = SnapWriter::new(&path).unwrap();
-            snap.finish().unwrap();
-        }
+            let snap = SnapWriter::new(dir.path(), None, Lsn(1234)).unwrap();
+            snap.finish().unwrap()
+        };
 
         {
             // Read the snapshot file.
+            let mut path = PathBuf::from(dir.path());
+            path.push(snap_meta.to_filename());
             let mut snap = SnapFile::new(&path).unwrap();
             assert_eq!(snap.page_index.page_count(), 0);
             assert!(!snap.has_page(0));

snapfile/src/versioned.rs

@@ -18,14 +18,44 @@ pub(crate) const SNAPFILE_MAGIC: u32 = 0x7fb8_38a8;
 // Constant chapter numbers
 // FIXME: the bookfile crate should use something better to index, e.g. strings.
 /// Snapshot-specific file metadata
-#[allow(dead_code)] // FIXME: this is a placeholder for future functionality.
 pub(crate) const CHAPTER_SNAP_META: u64 = 1;
 /// A packed set of 8KB pages.
 pub(crate) const CHAPTER_PAGES: u64 = 2;
 /// An index of pages.
 pub(crate) const CHAPTER_PAGE_INDEX: u64 = 3;
 
-// FIXME: move serialized data structs to a separate file.
+/// Information about the predecessor snapshot.
+///
+/// It contains the snap_id of the predecessor snapshot, and the LSN
+/// of that snapshot.
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
+pub struct Predecessor {
+    /// This is the id number of the previous snapshot.
+    ///
+    /// This must match the snap_id of the previous snapshot.
+    pub id: u64,
+
+    /// This is the LSN of the previous snapshot.
+    pub lsn: u64,
+}
+
+#[derive(Debug, PartialEq, Serialize, Deserialize, Versioned, UpgradeLatest)]
+pub struct SnapFileMetaV1 {
+    /// This is a unique ID number for this snapshot.
+    ///
+    /// This number guarantees that snapshot history is unique.
+    pub snap_id: u64,
+
+    /// Information about the predecessor snapshot.
+    ///
+    /// If `None`, this snapshot is the start of a new database.
+    pub predecessor: Option<Predecessor>,
+
+    /// This is the last LSN stored in this snapshot.
+    pub lsn: u64,
+}
+
+pub type SnapFileMeta = SnapFileMetaV1;
 
 /// An index from page number to offset within the pages chapter.
 #[derive(Debug, Default, Serialize, Deserialize, Versioned, UpgradeLatest)]
@@ -40,4 +70,5 @@ pub type PageIndex = PageIndexV1;
 // Each message gets a unique message id, for tracking by the aversion traits.
 assign_message_ids! {
     PageIndex: 100,
+    SnapFileMeta: 101,
 }