feat: region manifest checkpoint (#1202)

* chore: adds log when manifest protocol is changed

* chore: refactor region manifest

* temp commit

* feat: impl region manifest checkpoint

* feat: recover region version from manifest snapshot

* test: adds region snapshot test

* test: region manifest checkpoint

* test: alter region with manifest checkpoint

* fix: revert storage api

* feat: delete old snapshot

* refactor: manifest log storage

* Update src/storage/src/version.rs

Co-authored-by: Ruihang Xia <waynestxia@gmail.com>

* Update src/storage/src/manifest/checkpoint.rs

Co-authored-by: Ruihang Xia <waynestxia@gmail.com>

* Update src/storage/src/manifest/region.rs

Co-authored-by: Ruihang Xia <waynestxia@gmail.com>

* Update src/storage/src/manifest/region.rs

Co-authored-by: Ruihang Xia <waynestxia@gmail.com>

* chore: by CR comments

* refactor: by CR comments

* fix: typo

* chore: tweak start_version

---------

Co-authored-by: Ruihang Xia <waynestxia@gmail.com>

src/store-api/src/manifest.rs

@@ -28,6 +28,7 @@ pub type ManifestVersion = u64;
 pub const MIN_VERSION: u64 = 0;
 pub const MAX_VERSION: u64 = u64::MAX;
 
+/// The action to alter metadata
 pub trait MetaAction: Serialize + DeserializeOwned + Send + Sync + Clone + std::fmt::Debug {
     type Error: ErrorExt + Send + Sync;
 
@@ -47,6 +48,23 @@ pub trait MetaAction: Serialize + DeserializeOwned + Send + Sync + Clone + std::
         reader_version: ProtocolVersion,
     ) -> Result<(Self, Option<ProtocolAction>), Self::Error>;
 }
+/// The checkpoint by checkpoint
+pub trait Checkpoint: Send + Sync + Clone + std::fmt::Debug {
+    type Error: ErrorExt + Send + Sync;
+
+    /// Set a protocol action into checkpoint
+    fn set_protocol(&mut self, action: ProtocolAction);
+
+    /// The last compacted action's version of checkpoint
+    fn last_version(&self) -> ManifestVersion;
+
+    /// Encode this checkpoint into a byte vector
+    fn encode(&self) -> Result<Vec<u8>, Self::Error>;
+
+    /// Decode self from byte slice with reader protocol version,
+    /// return error when reader version is not supported.
+    fn decode(bs: &[u8], reader_version: ProtocolVersion) -> Result<Self, Self::Error>;
+}
 
 #[async_trait]
 pub trait MetaActionIterator {
@@ -64,6 +82,7 @@ pub trait Manifest: Send + Sync + Clone + 'static {
     type Error: ErrorExt + Send + Sync;
     type MetaAction: MetaAction;
     type MetaActionIterator: MetaActionIterator<Error = Self::Error, MetaAction = Self::MetaAction>;
+    type Checkpoint: Checkpoint;
 
     /// Update metadata by the action
     async fn update(&self, action: Self::MetaAction) -> Result<ManifestVersion, Self::Error>;
@@ -75,7 +94,12 @@ pub trait Manifest: Send + Sync + Clone + 'static {
         end: ManifestVersion,
     ) -> Result<Self::MetaActionIterator, Self::Error>;
 
-    async fn checkpoint(&self) -> Result<ManifestVersion, Self::Error>;
+    /// Do a checkpoint, it will create a checkpoint and compact actions.
+    async fn do_checkpoint(&self) -> Result<Option<Self::Checkpoint>, Self::Error>;
 
+    /// Returns the last success checkpoint
+    async fn last_checkpoint(&self) -> Result<Option<Self::Checkpoint>, Self::Error>;
+
+    /// Returns the last(or latest) manifest version.
     fn last_version(&self) -> ManifestVersion;
 }

src/store-api/src/manifest/action.rs

@@ -39,6 +39,16 @@ pub struct ProtocolAction {
     pub min_writer_version: ProtocolVersion,
 }
 
+impl std::fmt::Display for ProtocolAction {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "Protocol({}, {})",
+            &self.min_reader_version, &self.min_writer_version,
+        )
+    }
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
 pub struct VersionHeader {
     pub prev_version: ManifestVersion,

src/store-api/src/manifest/storage.rs

@@ -51,5 +51,14 @@ pub trait ManifestLogStorage {
     ) -> Result<(), Self::Error>;
 
     /// Load the latest checkpoint
-    async fn load_checkpoint(&self) -> Result<Option<(ManifestVersion, Vec<u8>)>, Self::Error>;
+    async fn load_last_checkpoint(&self)
+        -> Result<Option<(ManifestVersion, Vec<u8>)>, Self::Error>;
+    /// Delete the checkpoint by version
+    async fn delete_checkpoint(&self, version: ManifestVersion) -> Result<(), Self::Error>;
+
+    /// Load the checkpoint by version
+    async fn load_checkpoint(
+        &self,
+        version: ManifestVersion,
+    ) -> Result<Option<(ManifestVersion, Vec<u8>)>, Self::Error>;
 }