feat: impl table manifest (#157)

* feat: impl TableManifest and refactor table engine, object store etc.

* feat: persist table metadata when creating it

* fix: remove unused file src/storage/src/manifest/impl.rs

* feat: impl recover table info from manifest

* test: add open table test and table manifest test

* fix: resolve CR problems

* fix: compile error and remove region id

* doc: describe parent_dir

* fix: address CR problems

* fix: typo

* Revert "fix: compile error and remove region id"

This reverts commit c14c250f8a.

* fix: compile error and generate region id by table_id and region number

src/datanode/Cargo.toml

@@ -18,6 +18,7 @@ datatypes = { path = "../datatypes"}
 hyper = { version = "0.14", features = ["full"] }
 log-store = { path = "../log-store" }
 metrics = "0.18"
+object-store = { path = "../object-store" }
 query = { path = "../query" }
 serde = "1.0"
 serde_json = "1.0"

src/datanode/src/datanode.rs

@@ -4,11 +4,37 @@ use crate::error::Result;
 use crate::instance::{Instance, InstanceRef};
 use crate::server::Services;
 
+#[derive(Debug, Clone)]
+pub struct FileStoreConfig {
+    /// Storage path
+    pub store_dir: String,
+}
+
+impl Default for FileStoreConfig {
+    fn default() -> Self {
+        Self {
+            store_dir: "/tmp/greptimedb/data/".to_string(),
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub enum ObjectStoreConfig {
+    File(FileStoreConfig),
+}
+
+impl Default for ObjectStoreConfig {
+    fn default() -> Self {
+        ObjectStoreConfig::File(FileStoreConfig::default())
+    }
+}
+
 #[derive(Clone, Debug)]
 pub struct DatanodeOptions {
     pub http_addr: String,
     pub rpc_addr: String,
     pub wal_dir: String,
+    pub store_config: ObjectStoreConfig,
 }
 
 impl Default for DatanodeOptions {
@@ -17,6 +43,7 @@ impl Default for DatanodeOptions {
             http_addr: Default::default(),
             rpc_addr: Default::default(),
             wal_dir: "/tmp/wal".to_string(),
+            store_config: ObjectStoreConfig::default(),
         }
     }
 }

src/datanode/src/error.rs

@@ -108,6 +108,13 @@ pub enum Error {
 
     #[snafu(display("Failed to storage engine, source: {}", source))]
     OpenStorageEngine { source: StorageError },
+
+    #[snafu(display("Failed to init backend, dir: {}, source: {}", dir, source))]
+    InitBackend {
+        dir: String,
+        source: std::io::Error,
+        backtrace: Backtrace,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -133,6 +140,7 @@ impl ErrorExt for Error {
             | Error::TcpBind { .. }
             | Error::StartGrpc { .. }
             | Error::CreateDir { .. } => StatusCode::Internal,
+            Error::InitBackend { .. } => StatusCode::StorageUnavailable,
             Error::OpenLogStore { source } => source.status_code(),
             Error::OpenStorageEngine { source } => source.status_code(),
         }

src/datanode/src/instance.rs

@@ -6,16 +6,18 @@ use common_telemetry::logging::info;
 use datatypes::prelude::ConcreteDataType;
 use datatypes::schema::{ColumnSchema, Schema};
 use log_store::fs::{config::LogConfig, log::LocalFileLogStore};
+use object_store::{backend::fs::Backend, util, ObjectStore};
 use query::query_engine::{Output, QueryEngineFactory, QueryEngineRef};
 use snafu::{OptionExt, ResultExt};
 use sql::statements::statement::Statement;
-use storage::{config::EngineConfig, EngineImpl};
+use storage::{config::EngineConfig as StorageEngineConfig, EngineImpl};
 use table::engine::EngineContext;
 use table::engine::TableEngine;
 use table::requests::CreateTableRequest;
+use table_engine::config::EngineConfig as TableEngineConfig;
 use table_engine::engine::MitoEngine;
 
-use crate::datanode::DatanodeOptions;
+use crate::datanode::{DatanodeOptions, ObjectStoreConfig};
 use crate::error::{
     self, CreateTableSnafu, ExecuteSqlSnafu, InsertSnafu, NewCatalogSnafu, Result,
     TableNotFoundSnafu,
@@ -39,11 +41,17 @@ pub type InstanceRef = Arc<Instance>;
 
 impl Instance {
     pub async fn new(opts: &DatanodeOptions) -> Result<Self> {
+        let object_store = new_object_store(&opts.store_config).await?;
         let log_store = create_local_file_log_store(opts).await?;
+
         let table_engine = DefaultEngine::new(
-            EngineImpl::new(EngineConfig::default(), Arc::new(log_store))
-                .await
-                .context(error::OpenStorageEngineSnafu)?,
+            TableEngineConfig::default(),
+            EngineImpl::new(
+                StorageEngineConfig::default(),
+                Arc::new(log_store),
+                object_store.clone(),
+            ),
+            object_store,
         );
         let catalog_manager = Arc::new(
             catalog::LocalCatalogManager::try_new(Arc::new(table_engine.clone()))
@@ -138,6 +146,7 @@ impl Instance {
             .create_table(
                 &EngineContext::default(),
                 CreateTableRequest {
+                    id: 1,
                     name: table_name.to_string(),
                     desc: Some(" a test table".to_string()),
                     schema: Arc::new(
@@ -166,6 +175,26 @@ impl Instance {
     }
 }
 
+async fn new_object_store(store_config: &ObjectStoreConfig) -> Result<ObjectStore> {
+    // TODO(dennis): supports other backend
+    let store_dir = util::normalize_dir(match store_config {
+        ObjectStoreConfig::File(file) => &file.store_dir,
+    });
+
+    fs::create_dir_all(path::Path::new(&store_dir))
+        .context(error::CreateDirSnafu { dir: &store_dir })?;
+
+    info!("The storage directory is: {}", &store_dir);
+
+    let accessor = Backend::build()
+        .root(&store_dir)
+        .finish()
+        .await
+        .context(error::InitBackendSnafu { dir: &store_dir })?;
+
+    Ok(ObjectStore::new(accessor))
+}
+
 async fn create_local_file_log_store(opts: &DatanodeOptions) -> Result<LocalFileLogStore> {
     // create WAL directory
     fs::create_dir_all(path::Path::new(&opts.wal_dir))
@@ -196,7 +225,7 @@ mod tests {
     #[tokio::test]
     async fn test_execute_insert() {
         common_telemetry::init_default_ut_logging();
-        let (opts, _tmp_dir) = test_util::create_tmp_dir_and_datanode_opts();
+        let (opts, _guard) = test_util::create_tmp_dir_and_datanode_opts();
         let instance = Instance::new(&opts).await.unwrap();
         instance.start().await.unwrap();
 
@@ -215,7 +244,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_execute_query() {
-        let (opts, _tmp_dir) = test_util::create_tmp_dir_and_datanode_opts();
+        let (opts, _guard) = test_util::create_tmp_dir_and_datanode_opts();
         let instance = Instance::new(&opts).await.unwrap();
         instance.start().await.unwrap();
 

src/datanode/src/server/http/handler.rs

@@ -47,7 +47,7 @@ mod tests {
     use super::*;
     use crate::instance::Instance;
     use crate::server::http::JsonOutput;
-    use crate::test_util;
+    use crate::test_util::{self, TestGuard};
 
     fn create_params() -> Query<HashMap<String, String>> {
         let mut map = HashMap::new();
@@ -58,16 +58,16 @@ mod tests {
         Query(map)
     }
 
-    async fn create_extension() -> Extension<InstanceRef> {
-        let (opts, _tmp_dir) = test_util::create_tmp_dir_and_datanode_opts();
+    async fn create_extension() -> (Extension<InstanceRef>, TestGuard) {
+        let (opts, guard) = test_util::create_tmp_dir_and_datanode_opts();
         let instance = Arc::new(Instance::new(&opts).await.unwrap());
         instance.start().await.unwrap();
-        Extension(instance)
+        (Extension(instance), guard)
     }
 
     #[tokio::test]
     async fn test_sql_not_provided() {
-        let extension = create_extension().await;
+        let (extension, _guard) = create_extension().await;
 
         let json = sql(extension, Query(HashMap::default())).await;
         match json {
@@ -84,7 +84,7 @@ mod tests {
     async fn test_sql_output_rows() {
         common_telemetry::init_default_ut_logging();
         let query = create_params();
-        let extension = create_extension().await;
+        let (extension, _guard) = create_extension().await;
 
         let json = sql(extension, query).await;
 
@@ -112,7 +112,7 @@ mod tests {
         counter!("test_metrics", 1);
 
         let query = create_params();
-        let extension = create_extension().await;
+        let (extension, _guard) = create_extension().await;
         let text = metrics(extension, query).await;
 
         match text {

src/datanode/src/sql.rs

@@ -65,11 +65,13 @@ mod tests {
     use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
     use datatypes::value::Value;
     use log_store::fs::noop::NoopLogStore;
+    use object_store::{backend::fs::Backend, ObjectStore};
     use query::QueryEngineFactory;
-    use storage::config::EngineConfig;
+    use storage::config::EngineConfig as StorageEngineConfig;
     use storage::EngineImpl;
     use table::error::Result as TableResult;
     use table::{Table, TableRef};
+    use table_engine::config::EngineConfig as TableEngineConfig;
     use table_engine::engine::MitoEngine;
     use tempdir::TempDir;
 
@@ -138,6 +140,8 @@ mod tests {
     async fn test_statement_to_request() {
         let dir = TempDir::new("setup_test_engine_and_table").unwrap();
         let store_dir = dir.path().to_string_lossy();
+        let accessor = Backend::build().root(&store_dir).finish().await.unwrap();
+        let object_store = ObjectStore::new(accessor);
 
         let catalog_list = catalog::memory::new_memory_catalog_list().unwrap();
         let factory = QueryEngineFactory::new(catalog_list);
@@ -149,12 +153,13 @@ mod tests {
                            "#;
 
         let table_engine = MitoEngine::<EngineImpl<NoopLogStore>>::new(
+            TableEngineConfig::default(),
             EngineImpl::new(
-                EngineConfig::with_store_dir(&store_dir),
+                StorageEngineConfig::default(),
                 Arc::new(NoopLogStore::default()),
-            )
-            .await
-            .unwrap(),
+                object_store.clone(),
+            ),
+            object_store,
         );
         let sql_handler = SqlHandler::new(table_engine);
 

src/datanode/src/test_util.rs

@@ -1,17 +1,32 @@
 use tempdir::TempDir;
 
-use crate::datanode::DatanodeOptions;
+use crate::datanode::{DatanodeOptions, FileStoreConfig, ObjectStoreConfig};
 
 /// Create a tmp dir(will be deleted once it goes out of scope.) and a default `DatanodeOptions`,
 /// Only for test.
 ///
 /// TODO: Add a test feature
-pub fn create_tmp_dir_and_datanode_opts() -> (DatanodeOptions, TempDir) {
-    let tmp_dir = TempDir::new("/tmp/greptimedb_test").unwrap();
+pub struct TestGuard {
+    _wal_tmp_dir: TempDir,
+    _data_tmp_dir: TempDir,
+}
+
+pub fn create_tmp_dir_and_datanode_opts() -> (DatanodeOptions, TestGuard) {
+    let wal_tmp_dir = TempDir::new("/tmp/greptimedb_test_wal").unwrap();
+    let data_tmp_dir = TempDir::new("/tmp/greptimedb_test_data").unwrap();
     let opts = DatanodeOptions {
-        wal_dir: tmp_dir.path().to_str().unwrap().to_string(),
+        wal_dir: wal_tmp_dir.path().to_str().unwrap().to_string(),
+        store_config: ObjectStoreConfig::File(FileStoreConfig {
+            store_dir: data_tmp_dir.path().to_str().unwrap().to_string(),
+        }),
         ..Default::default()
     };
 
-    (opts, tmp_dir)
+    (
+        opts,
+        TestGuard {
+            _wal_tmp_dir: wal_tmp_dir,
+            _data_tmp_dir: data_tmp_dir,
+        },
+    )
 }

src/datanode/src/tests/http_test.rs

@@ -8,20 +8,20 @@ use axum_test_helper::TestClient;
 
 use crate::instance::Instance;
 use crate::server::http::HttpServer;
-use crate::test_util;
+use crate::test_util::{self, TestGuard};
 
-async fn make_test_app() -> Router {
-    let (opts, _tmp_dir) = test_util::create_tmp_dir_and_datanode_opts();
+async fn make_test_app() -> (Router, TestGuard) {
+    let (opts, guard) = test_util::create_tmp_dir_and_datanode_opts();
     let instance = Arc::new(Instance::new(&opts).await.unwrap());
     instance.start().await.unwrap();
     let http_server = HttpServer::new(instance);
-    http_server.make_app()
+    (http_server.make_app(), guard)
 }
 
 #[tokio::test]
 async fn test_sql_api() {
     common_telemetry::init_default_ut_logging();
-    let app = make_test_app().await;
+    let (app, _guard) = make_test_app().await;
     let client = TestClient::new(app);
     let res = client.get("/sql").send().await;
     assert_eq!(res.status(), StatusCode::OK);
@@ -49,7 +49,7 @@ async fn test_sql_api() {
 async fn test_metrics_api() {
     common_telemetry::init_default_ut_logging();
     common_telemetry::init_default_metrics_recorder();
-    let app = make_test_app().await;
+    let (app, _guard) = make_test_app().await;
     let client = TestClient::new(app);
 
     // Send a sql