From 8d2b517359d3fd69826ba1657c92f6f4b9d04b31 Mon Sep 17 00:00:00 2001 From: Eric Seppanen Date: Mon, 5 Jul 2021 21:49:06 -0700 Subject: [PATCH] snapfile: split apart code into multiple files versioned.rs: for things that get serialized and must be versioned to avoid breaking backwards compatibility. page.rs: for the Page struct. --- snapfile/src/lib.rs | 84 ++++++--------------------------------- snapfile/src/page.rs | 32 +++++++++++++++ snapfile/src/versioned.rs | 43 ++++++++++++++++++++ 3 files changed, 87 insertions(+), 72 deletions(-) create mode 100644 snapfile/src/page.rs create mode 100644 snapfile/src/versioned.rs diff --git a/snapfile/src/lib.rs b/snapfile/src/lib.rs index 509e10bb4d..4b63e3cbb3 100644 --- a/snapfile/src/lib.rs +++ b/snapfile/src/lib.rs @@ -4,41 +4,19 @@ #![forbid(unsafe_code)] #![warn(clippy::cast_possible_truncation)] +mod page; +mod versioned; +#[doc(inline)] +pub use page::Page; + use anyhow::{anyhow, bail, Result}; use aversion::group::{DataSink, DataSourceExt}; use aversion::util::cbor::CborData; -use aversion::{assign_message_ids, UpgradeLatest, Versioned}; use bookfile::{Book, BookWriter, ChapterIndex, ChapterWriter}; -use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; use std::fs::File; use std::io::Write; use std::path::Path; - -// A random constant, to identify this file type. -const SNAPFILE_MAGIC: u32 = 0x7fb8_38a8; - -// Constant chapter numbers -// FIXME: the bookfile crate should use something better to index, e.g. strings. -/// Snapshot-specific file metadata -#[allow(dead_code)] // FIXME: this is a placeholder for future functionality. -const CHAPTER_SNAP_META: u64 = 1; -/// A packed set of 8KB pages. -const CHAPTER_PAGES: u64 = 2; -/// An index of pages. -const CHAPTER_PAGE_INDEX: u64 = 3; - -// FIXME: move serialized data structs to a separate file. - -/// An index from page number to offset within the pages chapter. -#[derive(Debug, Default, Serialize, Deserialize, Versioned, UpgradeLatest)] -pub struct PageIndexV1 { - /// A map from page number to file offset. - map: BTreeMap, -} - -// A placeholder type, that will always point to the latest version. -type PageIndex = PageIndexV1; +use versioned::PageIndex; impl PageIndex { /// Retrieve the page offset from the index. @@ -53,44 +31,6 @@ impl PageIndex { } } -// Each message gets a unique message id, for tracking by the aversion traits. -assign_message_ids! { - PageIndex: 100, -} - -/// A single 8KB page. -pub struct Page(pub Box<[u8; 8192]>); - -impl Default for Page { - fn default() -> Self { - Page(Box::new([0u8; 8192])) - } -} - -impl From<[u8; 8192]> for Page { - fn from(array: [u8; 8192]) -> Self { - Page(Box::new(array)) - } -} - -impl From> for Page { - fn from(heap_array: Box<[u8; 8192]>) -> Self { - Page(heap_array) - } -} - -impl AsRef<[u8; 8192]> for Page { - fn as_ref(&self) -> &[u8; 8192] { - self.0.as_ref() - } -} - -impl AsMut<[u8; 8192]> for Page { - fn as_mut(&mut self) -> &mut [u8; 8192] { - self.0.as_mut() - } -} - /// A read-only snapshot file. pub struct SnapFile { book: Book, @@ -106,19 +46,19 @@ impl SnapFile { pub fn new(path: &Path) -> Result { let file = File::open(path)?; let mut book = Book::new(file)?; - if book.magic() != SNAPFILE_MAGIC { + if book.magic() != versioned::SNAPFILE_MAGIC { bail!("bad magic number"); } // Read the page index into memory. let chapter_num = book - .find_chapter(CHAPTER_PAGE_INDEX) + .find_chapter(versioned::CHAPTER_PAGE_INDEX) .ok_or_else(|| anyhow!("snapfile missing index chapter"))?; let chapter_reader = book.chapter_reader(chapter_num)?; let mut source = CborData::new(chapter_reader); let page_index: PageIndex = source.expect_message()?; let page_chapter_num = book - .find_chapter(CHAPTER_PAGES) + .find_chapter(versioned::CHAPTER_PAGES) .ok_or_else(|| anyhow!("snapfile missing pages chapter"))?; Ok(SnapFile { book, @@ -176,8 +116,8 @@ impl SnapWriter { /// Create a new `SnapWriter` pub fn new(path: &Path) -> Result { let file = File::create(path)?; - let book = BookWriter::new(file, SNAPFILE_MAGIC)?; - let writer = book.new_chapter(CHAPTER_PAGES); + let book = BookWriter::new(file, versioned::SNAPFILE_MAGIC)?; + let writer = book.new_chapter(versioned::CHAPTER_PAGES); Ok(SnapWriter { writer, page_index: PageIndex::default(), @@ -211,7 +151,7 @@ impl SnapWriter { // necessary file metadata. // FIXME: these 3 lines could be combined into a single function // that means "serialize this data structure with this format into this chapter". - let writer = book.new_chapter(CHAPTER_PAGE_INDEX); + let writer = book.new_chapter(versioned::CHAPTER_PAGE_INDEX); let mut sink = CborData::new(writer); sink.write_message(&self.page_index)?; diff --git a/snapfile/src/page.rs b/snapfile/src/page.rs new file mode 100644 index 0000000000..625dd64b8f --- /dev/null +++ b/snapfile/src/page.rs @@ -0,0 +1,32 @@ +/// A single 8KB page. +pub struct Page(pub Box<[u8; 8192]>); + +impl Default for Page { + fn default() -> Self { + Page(Box::new([0u8; 8192])) + } +} + +impl From<[u8; 8192]> for Page { + fn from(array: [u8; 8192]) -> Self { + Page(Box::new(array)) + } +} + +impl From> for Page { + fn from(heap_array: Box<[u8; 8192]>) -> Self { + Page(heap_array) + } +} + +impl AsRef<[u8; 8192]> for Page { + fn as_ref(&self) -> &[u8; 8192] { + self.0.as_ref() + } +} + +impl AsMut<[u8; 8192]> for Page { + fn as_mut(&mut self) -> &mut [u8; 8192] { + self.0.as_mut() + } +} diff --git a/snapfile/src/versioned.rs b/snapfile/src/versioned.rs new file mode 100644 index 0000000000..91a7f9ce73 --- /dev/null +++ b/snapfile/src/versioned.rs @@ -0,0 +1,43 @@ +//! Versioned data structures for snapshot files +//! +//! To ensure that future versions of software can read snapshot files, +//! all data structures that are serialized into the snapshot files should +//! live in this module. +//! +//! Once released, versioned data structures should never be modified. +//! Instead, new versions should be created and conversion functions should +//! be defined using the `FromVersion` trait. + +use aversion::{assign_message_ids, UpgradeLatest, Versioned}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +// A random constant, to identify this file type. +pub(crate) const SNAPFILE_MAGIC: u32 = 0x7fb8_38a8; + +// Constant chapter numbers +// FIXME: the bookfile crate should use something better to index, e.g. strings. +/// Snapshot-specific file metadata +#[allow(dead_code)] // FIXME: this is a placeholder for future functionality. +pub(crate) const CHAPTER_SNAP_META: u64 = 1; +/// A packed set of 8KB pages. +pub(crate) const CHAPTER_PAGES: u64 = 2; +/// An index of pages. +pub(crate) const CHAPTER_PAGE_INDEX: u64 = 3; + +// FIXME: move serialized data structs to a separate file. + +/// An index from page number to offset within the pages chapter. +#[derive(Debug, Default, Serialize, Deserialize, Versioned, UpgradeLatest)] +pub struct PageIndexV1 { + /// A map from page number to file offset. + pub(crate) map: BTreeMap, +} + +// A placeholder type, that will always point to the latest version. +pub type PageIndex = PageIndexV1; + +// Each message gets a unique message id, for tracking by the aversion traits. +assign_message_ids! { + PageIndex: 100, +}