feat: impl table manifest (#157)

* feat: impl TableManifest and refactor table engine, object store etc.

* feat: persist table metadata when creating it

* fix: remove unused file src/storage/src/manifest/impl.rs

* feat: impl recover table info from manifest

* test: add open table test and table manifest test

* fix: resolve CR problems

* fix: compile error and remove region id

* doc: describe parent_dir

* fix: address CR problems

* fix: typo

* Revert "fix: compile error and remove region id"

This reverts commit c14c250f8a.

* fix: compile error and generate region id by table_id and region number

src/store-api/src/manifest.rs

@@ -6,14 +6,29 @@ use async_trait::async_trait;
 use common_error::ext::ErrorExt;
 use serde::{de::DeserializeOwned, Serialize};
 
+use crate::manifest::action::ProtocolAction;
+use crate::manifest::action::ProtocolVersion;
 pub use crate::manifest::storage::*;
 
 pub type ManifestVersion = u64;
 pub const MIN_VERSION: u64 = 0;
 pub const MAX_VERSION: u64 = u64::MAX;
 
-pub trait MetaAction: Serialize + DeserializeOwned {
+pub trait MetaAction: Serialize + DeserializeOwned + Send + Sync + Clone + std::fmt::Debug {
+    type Error: ErrorExt + Send + Sync;
+
+    /// Set previous valid manifest version.
     fn set_prev_version(&mut self, version: ManifestVersion);
+
+    /// Encode this action into a byte vector
+    fn encode(&self) -> Result<Vec<u8>, Self::Error>;
+
+    /// Decode self from byte slice with reader protocol version,
+    /// return error when reader version is not supported.
+    fn decode(
+        bs: &[u8],
+        reader_version: ProtocolVersion,
+    ) -> Result<(Self, Option<ProtocolAction>), Self::Error>;
 }
 
 #[async_trait]

src/store-api/src/manifest/action.rs

@@ -1,6 +1,8 @@
 ///! Common actions for manifest
 use serde::{Deserialize, Serialize};
 
+use crate::manifest::ManifestVersion;
+
 pub type ProtocolVersion = u16;
 
 /// Current reader and writer versions
@@ -23,6 +25,11 @@ pub struct ProtocolAction {
     pub min_writer_version: ProtocolVersion,
 }
 
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
+pub struct VersionHeader {
+    pub prev_version: ManifestVersion,
+}
+
 impl Default for ProtocolAction {
     fn default() -> Self {
         let (min_reader_version, min_writer_version) = supported_protocol_version();

src/store-api/src/storage.rs

@@ -16,7 +16,7 @@ pub use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
 
 pub use self::chunk::{Chunk, ChunkReader};
 pub use self::descriptors::*;
-pub use self::engine::{EngineContext, OpenOptions, StorageEngine};
+pub use self::engine::{CreateOptions, EngineContext, OpenOptions, StorageEngine};
 pub use self::metadata::RegionMeta;
 pub use self::region::{Region, WriteContext};
 pub use self::requests::{GetRequest, PutOperation, ScanRequest, WriteRequest};

src/store-api/src/storage/descriptors.rs

@@ -9,7 +9,7 @@ pub type ColumnId = u32;
 /// Id of column family, unique in each region.
 pub type ColumnFamilyId = u32;
 /// Id of the region.
-pub type RegionId = u32;
+pub type RegionId = u64;
 
 // TODO(yingwen): Validate default value has same type with column, and name is a valid column name.
 /// A [ColumnDescriptor] contains information to create a column.

src/store-api/src/storage/engine.rs

@@ -39,6 +39,7 @@ pub trait StorageEngine: Send + Sync + Clone + 'static {
         &self,
         ctx: &EngineContext,
         descriptor: RegionDescriptor,
+        opts: &CreateOptions,
     ) -> Result<Self::Region, Self::Error>;
 
     /// Drops given region.
@@ -62,6 +63,16 @@ pub trait StorageEngine: Send + Sync + Clone + 'static {
 #[derive(Debug, Clone, Default)]
 pub struct EngineContext {}
 
+/// Options to create a region.
+#[derive(Debug, Clone, Default)]
+pub struct CreateOptions {
+    /// Region parent directory
+    pub parent_dir: String,
+}
+
 /// Options to open a region.
 #[derive(Debug, Clone, Default)]
-pub struct OpenOptions {}
+pub struct OpenOptions {
+    /// Region parent directory
+    pub parent_dir: String,
+}