diff --git a/Cargo.lock b/Cargo.lock index 9db13d21a6..0dda8f0818 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6166,6 +6166,32 @@ dependencies = [ "winapi", ] +[[package]] +name = "tests-integration" +version = "0.1.0" +dependencies = [ + "api", + "axum 0.6.0-rc.2", + "axum-test-helper", + "catalog", + "client", + "common-catalog", + "common-runtime", + "common-telemetry", + "datanode", + "datatypes", + "frontend", + "mito", + "serde", + "serde_json", + "servers", + "snafu", + "sql", + "table", + "tempdir", + "tokio", +] + [[package]] name = "textwrap" version = "0.11.0" diff --git a/Cargo.toml b/Cargo.toml index 99ce7831d3..678ef002ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,7 @@ members = [ "src/table", "src/mito", "tests/runner", + "tests-integration" ] [profile.release] diff --git a/src/datanode/src/lib.rs b/src/datanode/src/lib.rs index 8540c851f6..3e1aa92a76 100644 --- a/src/datanode/src/lib.rs +++ b/src/datanode/src/lib.rs @@ -22,6 +22,6 @@ mod metric; mod mock; mod script; pub mod server; -mod sql; +pub mod sql; #[cfg(test)] mod tests; diff --git a/src/datanode/src/tests.rs b/src/datanode/src/tests.rs index 5cb02b3453..8c460a53fd 100644 --- a/src/datanode/src/tests.rs +++ b/src/datanode/src/tests.rs @@ -12,7 +12,5 @@ // See the License for the specific language governing permissions and // limitations under the License. -mod grpc_test; -mod http_test; mod instance_test; pub(crate) mod test_util; diff --git a/src/servers/src/http.rs b/src/servers/src/http.rs index 3aced9a8cf..74e8d7f1c2 100644 --- a/src/servers/src/http.rs +++ b/src/servers/src/http.rs @@ -185,7 +185,7 @@ impl TryFrom> for HttpRecordsOutput { } } -#[derive(Serialize, Deserialize, Debug, JsonSchema)] +#[derive(Serialize, Deserialize, Debug, JsonSchema, Eq, PartialEq)] #[serde(rename_all = "lowercase")] pub enum JsonOutput { AffectedRows(usize), diff --git a/tests-integration/Cargo.toml b/tests-integration/Cargo.toml new file mode 100644 index 0000000000..332eb656e2 --- /dev/null +++ b/tests-integration/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "tests-integration" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" + +[dependencies] +api = { path = "../src/api" } +axum = "0.6.0-rc.2" +axum-test-helper = { git = "https://github.com/sunng87/axum-test-helper.git", branch = "patch-1" } +catalog = { path = "../src/catalog" } +client = { path = "../src/client" } +common-catalog = { path = "../src/common/catalog" } +common-runtime = { path = "../src/common/runtime" } +common-telemetry = { path = "../src/common/telemetry" } +datanode = { path = "../src/datanode" } +datatypes = { path = "../src/datatypes" } +frontend = { path = "../src/frontend" } +mito = { path = "../src/mito", features = ["test"] } +serde = "1.0" +serde_json = "1.0" +servers = { path = "../src/servers" } +snafu = { version = "0.7", features = ["backtraces"] } +sql = { path = "../src/sql" } +table = { path = "../src/table" } +tempdir = "0.3" +tokio = { version = "1.20", features = ["full"] } diff --git a/tests-integration/src/lib.rs b/tests-integration/src/lib.rs new file mode 100644 index 0000000000..1bfde512a8 --- /dev/null +++ b/tests-integration/src/lib.rs @@ -0,0 +1,15 @@ +// Copyright 2022 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod test_util; diff --git a/tests-integration/src/test_util.rs b/tests-integration/src/test_util.rs new file mode 100644 index 0000000000..9ae6ac7751 --- /dev/null +++ b/tests-integration/src/test_util.rs @@ -0,0 +1,107 @@ +// Copyright 2022 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashMap; +use std::sync::Arc; + +use catalog::CatalogManagerRef; +use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, MIN_USER_TABLE_ID}; +use datanode::datanode::{DatanodeOptions, ObjectStoreConfig}; +use datanode::error::{CreateTableSnafu, Result}; +use datanode::sql::SqlHandler; +use datatypes::data_type::ConcreteDataType; +use datatypes::schema::{ColumnSchema, SchemaBuilder}; +use servers::Mode; +use snafu::ResultExt; +use table::engine::{EngineContext, TableEngineRef}; +use table::requests::CreateTableRequest; +use tempdir::TempDir; + +/// Create a tmp dir(will be deleted once it goes out of scope.) and a default `DatanodeOptions`, +/// Only for test. +pub struct TestGuard { + _wal_tmp_dir: TempDir, + _data_tmp_dir: TempDir, +} + +pub fn create_tmp_dir_and_datanode_opts(name: &str) -> (DatanodeOptions, TestGuard) { + let wal_tmp_dir = TempDir::new(&format!("gt_wal_{}", name)).unwrap(); + let data_tmp_dir = TempDir::new(&format!("gt_data_{}", name)).unwrap(); + let opts = DatanodeOptions { + wal_dir: wal_tmp_dir.path().to_str().unwrap().to_string(), + storage: ObjectStoreConfig::File { + data_dir: data_tmp_dir.path().to_str().unwrap().to_string(), + }, + mode: Mode::Standalone, + ..Default::default() + }; + ( + opts, + TestGuard { + _wal_tmp_dir: wal_tmp_dir, + _data_tmp_dir: data_tmp_dir, + }, + ) +} + +pub async fn create_test_table( + catalog_manager: &CatalogManagerRef, + sql_handler: &SqlHandler, + ts_type: ConcreteDataType, +) -> Result<()> { + let column_schemas = vec![ + ColumnSchema::new("host", ConcreteDataType::string_datatype(), false), + ColumnSchema::new("cpu", ConcreteDataType::float64_datatype(), true), + ColumnSchema::new("memory", ConcreteDataType::float64_datatype(), true), + ColumnSchema::new("ts", ts_type, true).with_time_index(true), + ]; + + let table_name = "demo"; + let table_engine: TableEngineRef = sql_handler.table_engine(); + let table = table_engine + .create_table( + &EngineContext::default(), + CreateTableRequest { + id: MIN_USER_TABLE_ID, + catalog_name: "greptime".to_string(), + schema_name: "public".to_string(), + table_name: table_name.to_string(), + desc: Some(" a test table".to_string()), + schema: Arc::new( + SchemaBuilder::try_from(column_schemas) + .unwrap() + .build() + .expect("ts is expected to be timestamp column"), + ), + create_if_not_exists: true, + primary_key_indices: vec![3, 0], // "host" and "ts" are primary keys + table_options: HashMap::new(), + region_numbers: vec![0], + }, + ) + .await + .context(CreateTableSnafu { table_name })?; + + let schema_provider = catalog_manager + .catalog(DEFAULT_CATALOG_NAME) + .unwrap() + .unwrap() + .schema(DEFAULT_SCHEMA_NAME) + .unwrap() + .unwrap(); + schema_provider + .register_table(table_name.to_string(), table) + .unwrap(); + Ok(()) +} diff --git a/src/datanode/src/tests/grpc_test.rs b/tests-integration/tests/grpc.rs similarity index 98% rename from src/datanode/src/tests/grpc_test.rs rename to tests-integration/tests/grpc.rs index 5fefa42cba..0fc41addae 100644 --- a/src/datanode/src/tests/grpc_test.rs +++ b/tests-integration/tests/grpc.rs @@ -11,6 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +#![feature(assert_matches)] use std::assert_matches::assert_matches; use std::net::SocketAddr; @@ -27,14 +28,13 @@ use client::admin::Admin; use client::{Client, Database, ObjectResult}; use common_catalog::consts::MIN_USER_TABLE_ID; use common_runtime::Builder as RuntimeBuilder; +use datanode::instance::Instance; use frontend::frontend::FrontendOptions; use frontend::grpc::GrpcOptions; use servers::grpc::GrpcServer; use servers::server::Server; use servers::Mode; - -use crate::instance::Instance; -use crate::tests::test_util::{self, TestGuard}; +use tests_integration::test_util::{self, TestGuard}; async fn setup_grpc_server( name: &str, @@ -78,7 +78,7 @@ async fn setup_grpc_server( let mut fe_instance = frontend::instance::Instance::try_new(&fe_opts) .await .unwrap(); - fe_instance.set_catalog_manager(instance.catalog_manager.clone()); + fe_instance.set_catalog_manager(instance.catalog_manager().clone()); let fe_instance_ref = Arc::new(fe_instance); let fe_grpc_server = Arc::new(GrpcServer::new( diff --git a/src/datanode/src/tests/http_test.rs b/tests-integration/tests/http.rs similarity index 73% rename from src/datanode/src/tests/http_test.rs rename to tests-integration/tests/http.rs index 1dcdf9405a..712900a809 100644 --- a/src/datanode/src/tests/http_test.rs +++ b/tests-integration/tests/http.rs @@ -17,15 +17,14 @@ use std::sync::Arc; use axum::http::StatusCode; use axum::Router; use axum_test_helper::TestClient; +use datanode::instance::{Instance, InstanceRef}; use datatypes::prelude::ConcreteDataType; use frontend::frontend::FrontendOptions; use frontend::instance::{FrontendInstance, Instance as FeInstance}; use serde_json::json; -use servers::http::{ColumnSchema, HttpOptions, HttpServer, JsonOutput, JsonResponse, Schema}; +use servers::http::{HttpOptions, HttpServer, JsonOutput, JsonResponse}; use test_util::TestGuard; - -use crate::instance::{Instance, InstanceRef}; -use crate::tests::test_util; +use tests_integration::test_util; async fn build_frontend_instance(datanode_instance: InstanceRef) -> FeInstance { let fe_opts = FrontendOptions::default(); @@ -98,21 +97,12 @@ async fn test_sql_api() { let output = body.output().unwrap(); assert_eq!(output.len(), 1); - if let JsonOutput::Records(records) = &output[0] { - assert_eq!(records.num_cols(), 1); - assert_eq!(records.num_rows(), 10); - assert_eq!( - records.schema().unwrap(), - &Schema::new(vec![ColumnSchema::new( - "number".to_owned(), - "UInt32".to_owned() - )]) - ); - assert_eq!(records.rows()[0][0], json!(0)); - assert_eq!(records.rows()[9][0], json!(9)); - } else { - unreachable!() - } + assert_eq!( + output[0], + serde_json::from_value::(json!({ + "records" :{"schema":{"column_schemas":[{"name":"number","data_type":"UInt32"}]},"rows":[[0],[1],[2],[3],[4],[5],[6],[7],[8],[9]]} + })).unwrap() + ); // test insert and select let res = client @@ -134,25 +124,13 @@ async fn test_sql_api() { assert!(body.execution_time_ms().is_some()); let output = body.output().unwrap(); assert_eq!(output.len(), 1); - if let JsonOutput::Records(records) = &output[0] { - assert_eq!(records.num_cols(), 4); - assert_eq!(records.num_rows(), 1); - assert_eq!( - records.schema().unwrap(), - &Schema::new(vec![ - ColumnSchema::new("host".to_owned(), "String".to_owned()), - ColumnSchema::new("cpu".to_owned(), "Float64".to_owned()), - ColumnSchema::new("memory".to_owned(), "Float64".to_owned()), - ColumnSchema::new("ts".to_owned(), "Timestamp".to_owned()) - ]) - ); - assert_eq!( - records.rows()[0], - vec![json!("host"), json!(66.6), json!(1024.0), json!(0)] - ); - } else { - unreachable!(); - } + + assert_eq!( + output[0], + serde_json::from_value::(json!({ + "records":{"schema":{"column_schemas":[{"name":"host","data_type":"String"},{"name":"cpu","data_type":"Float64"},{"name":"memory","data_type":"Float64"},{"name":"ts","data_type":"Timestamp"}]},"rows":[["host",66.6,1024.0,0]]} + })).unwrap() + ); // select with projections let res = client @@ -168,20 +146,13 @@ async fn test_sql_api() { assert!(body.execution_time_ms().is_some()); let output = body.output().unwrap(); assert_eq!(output.len(), 1); - if let JsonOutput::Records(records) = &output[0] { - assert_eq!(records.num_cols(), 2); - assert_eq!(records.num_rows(), 1); - assert_eq!( - records.schema().unwrap(), - &Schema::new(vec![ - ColumnSchema::new("cpu".to_owned(), "Float64".to_owned()), - ColumnSchema::new("ts".to_owned(), "Timestamp".to_owned()) - ]) - ); - assert_eq!(records.rows()[0], vec![json!(66.6), json!(0)]); - } else { - unreachable!() - } + + assert_eq!( + output[0], + serde_json::from_value::(json!({ + "records":{"schema":{"column_schemas":[{"name":"cpu","data_type":"Float64"},{"name":"ts","data_type":"Timestamp"}]},"rows":[[66.6,0]]} + })).unwrap() + ); // select with column alias let res = client @@ -197,20 +168,12 @@ async fn test_sql_api() { assert!(body.execution_time_ms().is_some()); let output = body.output().unwrap(); assert_eq!(output.len(), 1); - if let JsonOutput::Records(records) = &output[0] { - assert_eq!(records.num_cols(), 2); - assert_eq!(records.num_rows(), 1); - assert_eq!( - records.schema().unwrap(), - &Schema::new(vec![ - ColumnSchema::new("c".to_owned(), "Float64".to_owned()), - ColumnSchema::new("time".to_owned(), "Timestamp".to_owned()) - ]) - ); - assert_eq!(records.rows()[0], vec![json!(66.6), json!(0)]); - } else { - unreachable!() - } + assert_eq!( + output[0], + serde_json::from_value::(json!({ + "records":{"schema":{"column_schemas":[{"name":"c","data_type":"Float64"},{"name":"time","data_type":"Timestamp"}]},"rows":[[66.6,0]]} + })).unwrap() + ); } #[tokio::test(flavor = "multi_thread")] @@ -269,18 +232,10 @@ def test(n): assert!(body.execution_time_ms().is_some()); let output = body.output().unwrap(); assert_eq!(output.len(), 1); - if let JsonOutput::Records(ref records) = output[0] { - assert_eq!(records.num_cols(), 1); - assert_eq!(records.num_rows(), 10); - assert_eq!( - records.schema().unwrap(), - &Schema::new(vec![ColumnSchema::new( - "n".to_owned(), - "Float64".to_owned() - )]) - ); - assert_eq!(records.rows()[0][0], json!(1.0)); - } else { - unreachable!() - } + assert_eq!( + output[0], + serde_json::from_value::(json!({ + "records":{"schema":{"column_schemas":[{"name":"n","data_type":"Float64"}]},"rows":[[1.0],[2.0],[3.0],[4.0],[5.0],[6.0],[7.0],[8.0],[9.0],[10.0]]} + })).unwrap() + ); }