new sstable format (#1943)

* document a new sstable format

* add support for changing target block size

* use new format for sstable index

* handle sstable version errror

* use very small blocks for proptests

* add a footer structure

common/src/dictionary_footer.rs

@@ -0,0 +1,63 @@
+use std::io::{self, Read, Write};
+
+use crate::BinarySerializable;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[repr(u32)]
+pub enum DictionaryKind {
+    Fst = 1,
+    SSTable = 2,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct DictionaryFooter {
+    pub kind: DictionaryKind,
+    pub version: u32,
+}
+
+impl DictionaryFooter {
+    pub fn verify_equal(&self, other: &DictionaryFooter) -> io::Result<()> {
+        if self.kind != other.kind {
+            return Err(io::Error::new(
+                io::ErrorKind::Other,
+                format!(
+                    "Invalid dictionary type, expected {:?}, found {:?}",
+                    self.kind, other.kind
+                ),
+            ));
+        }
+        if self.version != other.version {
+            return Err(io::Error::new(
+                io::ErrorKind::Other,
+                format!(
+                    "Unsuported dictionary version, expected {}, found {}",
+                    self.version, other.version
+                ),
+            ));
+        }
+        Ok(())
+    }
+}
+
+impl BinarySerializable for DictionaryFooter {
+    fn serialize<W: Write + ?Sized>(&self, writer: &mut W) -> io::Result<()> {
+        self.version.serialize(writer)?;
+        (self.kind as u32).serialize(writer)
+    }
+    fn deserialize<R: Read>(reader: &mut R) -> io::Result<Self> {
+        let version = u32::deserialize(reader)?;
+        let kind = u32::deserialize(reader)?;
+        let kind = match kind {
+            1 => DictionaryKind::Fst,
+            2 => DictionaryKind::SSTable,
+            _ => {
+                return Err(io::Error::new(
+                    io::ErrorKind::Other,
+                    format!("invalid dictionary kind: {kind}"),
+                ))
+            }
+        };
+
+        Ok(DictionaryFooter { kind, version })
+    }
+}

common/src/lib.rs

@@ -7,6 +7,7 @@ pub use byteorder::LittleEndian as Endianness;
 mod bitset;
 mod byte_count;
 mod datetime;
+mod dictionary_footer;
 pub mod file_slice;
 mod group_by;
 mod serialize;
@@ -15,6 +16,7 @@ mod writer;
 pub use bitset::*;
 pub use byte_count::ByteCount;
 pub use datetime::{DatePrecision, DateTime};
+pub use dictionary_footer::*;
 pub use group_by::GroupByIteratorExtended;
 pub use ownedbytes::{OwnedBytes, StableDeref};
 pub use serialize::{BinarySerializable, DeserializeFrom, FixedSize};