From b31a6cb506c5fd4cee417ddb3c0780d25272b505 Mon Sep 17 00:00:00 2001 From: Eugene Tolbakov Date: Wed, 8 Mar 2023 03:15:56 +0000 Subject: [PATCH] refactor: replace tempdir with tempfile (#1123) * refactor: replace tempdir with tempfile * refactor(query): move tempfile dependency under the workspace's Cargo.toml * refactor(tempfile): create common-test-util * refactor(tempfile): fix toml format * refactor(tempfile): remove tempfile out of dependencies * refactor(tempfile): fix incorrect toml --- Cargo.toml | 1 + src/catalog/Cargo.toml | 2 +- src/catalog/src/system.rs | 4 ++-- src/cmd/Cargo.toml | 3 +-- src/cmd/src/datanode.rs | 4 ++-- src/cmd/src/frontend.rs | 4 ++-- src/cmd/src/metasrv.rs | 4 ++-- src/cmd/src/toml_loader.rs | 4 ++-- src/cmd/tests/cli.rs | 6 ++--- src/common/catalog/Cargo.toml | 1 - src/common/procedure/Cargo.toml | 2 +- src/common/procedure/src/local.rs | 12 +++++----- src/common/procedure/src/local/runner.rs | 14 +++++------ src/common/procedure/src/store.rs | 10 ++++---- src/common/procedure/src/store/state_store.rs | 4 ++-- src/common/test-util/Cargo.toml | 8 +++++++ src/common/test-util/src/lib.rs | 15 ++++++++++++ src/common/test-util/src/temp_dir.rs | 23 +++++++++++++++++++ src/datanode/Cargo.toml | 2 +- src/datanode/src/sql.rs | 4 ++-- src/datanode/src/tests/test_util.rs | 8 +++---- src/frontend/Cargo.toml | 2 +- src/frontend/src/tests.rs | 10 ++++---- src/log-store/Cargo.toml | 2 +- src/log-store/src/raft_engine/log_store.rs | 14 +++++------ src/log-store/src/test_util/log_store_util.rs | 14 ++++------- src/mito/Cargo.toml | 6 ++--- src/mito/src/engine/procedure.rs | 2 +- src/mito/src/engine/tests.rs | 2 +- src/mito/src/table/test_util.rs | 4 ++-- src/object-store/Cargo.toml | 2 +- src/object-store/tests/object_store_test.rs | 10 ++++---- src/script/Cargo.toml | 2 +- src/script/src/manager.rs | 5 ++-- src/servers/Cargo.toml | 2 +- src/servers/src/auth/user_provider.rs | 4 ++-- src/storage/Cargo.toml | 2 +- src/storage/src/compaction/writer.rs | 6 ++--- src/storage/src/engine.rs | 9 ++++---- src/storage/src/file_purger.rs | 6 ++--- src/storage/src/manifest/region.rs | 4 ++-- src/storage/src/manifest/storage.rs | 4 ++-- src/storage/src/region/tests.rs | 13 ++++------- src/storage/src/region/tests/alter.rs | 10 ++++---- src/storage/src/region/tests/basic.rs | 16 ++++++------- src/storage/src/region/tests/close.rs | 6 ++--- src/storage/src/region/tests/flush.rs | 10 ++++---- src/storage/src/region/tests/projection.rs | 6 ++--- src/storage/src/sst/parquet.rs | 10 ++++---- src/storage/src/wal.rs | 13 +++++++---- src/table-procedure/Cargo.toml | 2 +- src/table-procedure/src/test_util.rs | 4 ++-- src/table/Cargo.toml | 2 +- src/table/src/predicate.rs | 4 ++-- tests-integration/Cargo.toml | 3 ++- tests-integration/src/test_util.rs | 6 ++--- 56 files changed, 198 insertions(+), 154 deletions(-) create mode 100644 src/common/test-util/Cargo.toml create mode 100644 src/common/test-util/src/lib.rs create mode 100644 src/common/test-util/src/temp_dir.rs diff --git a/Cargo.toml b/Cargo.toml index 27bb3f4dfc..7c62f0498b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ members = [ "src/common/runtime", "src/common/substrait", "src/common/telemetry", + "src/common/test-util", "src/common/time", "src/datanode", "src/datatypes", diff --git a/src/catalog/Cargo.toml b/src/catalog/Cargo.toml index 8ce18e5198..8492ab44ef 100644 --- a/src/catalog/Cargo.toml +++ b/src/catalog/Cargo.toml @@ -36,10 +36,10 @@ table = { path = "../table" } tokio.workspace = true [dev-dependencies] +common-test-util = { path = "../common/test-util" } chrono.workspace = true log-store = { path = "../log-store" } mito = { path = "../mito", features = ["test"] } object-store = { path = "../object-store" } storage = { path = "../storage" } -tempdir = "0.3" tokio.workspace = true diff --git a/src/catalog/src/system.rs b/src/catalog/src/system.rs index 19dffa492d..1b86529635 100644 --- a/src/catalog/src/system.rs +++ b/src/catalog/src/system.rs @@ -395,6 +395,7 @@ pub struct TableEntryValue { #[cfg(test)] mod tests { use common_recordbatch::RecordBatches; + use common_test_util::temp_dir::{create_temp_dir, TempDir}; use datatypes::value::Value; use log_store::NoopLogStore; use mito::config::EngineConfig; @@ -405,7 +406,6 @@ mod tests { use storage::EngineImpl; use table::metadata::TableType; use table::metadata::TableType::Base; - use tempdir::TempDir; use super::*; @@ -480,7 +480,7 @@ mod tests { } pub async fn prepare_table_engine() -> (TempDir, TableEngineRef) { - let dir = TempDir::new("system-table-test").unwrap(); + let dir = create_temp_dir("system-table-test"); let store_dir = dir.path().to_string_lossy(); let accessor = object_store::services::Fs::default() .root(&store_dir) diff --git a/src/cmd/Cargo.toml b/src/cmd/Cargo.toml index fb84255a74..9c5edc09bd 100644 --- a/src/cmd/Cargo.toml +++ b/src/cmd/Cargo.toml @@ -46,10 +46,9 @@ toml = "0.5" [dev-dependencies] +common-test-util = { path = "../common/test-util" } rexpect = "0.5" serde.workspace = true -tempdir = "0.3" -tempfile.workspace = true [build-dependencies] build-data = "0.1.3" diff --git a/src/cmd/src/datanode.rs b/src/cmd/src/datanode.rs index 14c4c2da39..405dc36f5d 100644 --- a/src/cmd/src/datanode.rs +++ b/src/cmd/src/datanode.rs @@ -153,15 +153,15 @@ mod tests { use std::io::Write; use std::time::Duration; + use common_test_util::temp_dir::create_named_temp_file; use datanode::datanode::{CompactionConfig, ObjectStoreConfig}; use servers::Mode; - use tempfile::NamedTempFile; use super::*; #[test] fn test_read_from_config_file() { - let mut file = NamedTempFile::new().unwrap(); + let mut file = create_named_temp_file(); let toml_str = r#" mode = "distributed" enable_memory_catalog = false diff --git a/src/cmd/src/frontend.rs b/src/cmd/src/frontend.rs index 2c54a1e896..b89657abd7 100644 --- a/src/cmd/src/frontend.rs +++ b/src/cmd/src/frontend.rs @@ -182,8 +182,8 @@ mod tests { use std::io::Write; use std::time::Duration; + use common_test_util::temp_dir::create_named_temp_file; use servers::auth::{Identity, Password, UserProviderRef}; - use tempfile::NamedTempFile; use super::*; @@ -241,7 +241,7 @@ mod tests { #[test] fn test_read_from_config_file() { - let mut file = NamedTempFile::new().unwrap(); + let mut file = create_named_temp_file(); let toml_str = r#" mode = "distributed" diff --git a/src/cmd/src/metasrv.rs b/src/cmd/src/metasrv.rs index 0d909bf200..8151d3b3f4 100644 --- a/src/cmd/src/metasrv.rs +++ b/src/cmd/src/metasrv.rs @@ -115,8 +115,8 @@ impl TryFrom for MetaSrvOptions { mod tests { use std::io::Write; + use common_test_util::temp_dir::create_named_temp_file; use meta_srv::selector::SelectorType; - use tempfile::NamedTempFile; use super::*; @@ -139,7 +139,7 @@ mod tests { #[test] fn test_read_from_config_file() { - let mut file = NamedTempFile::new().unwrap(); + let mut file = create_named_temp_file(); let toml_str = r#" bind_addr = "127.0.0.1:3002" server_addr = "127.0.0.1:3002" diff --git a/src/cmd/src/toml_loader.rs b/src/cmd/src/toml_loader.rs index 2f1ba739a5..9d689e50bc 100644 --- a/src/cmd/src/toml_loader.rs +++ b/src/cmd/src/toml_loader.rs @@ -29,9 +29,9 @@ mod tests { use std::fs::File; use std::io::Write; + use common_test_util::temp_dir::create_temp_dir; use serde::{Deserialize, Serialize}; use snafu::ResultExt; - use tempdir::TempDir; use super::*; use crate::error::Result; @@ -62,7 +62,7 @@ mod tests { host: "greptime.test".to_string(), }; - let dir = TempDir::new("test_from_file").unwrap(); + let dir = create_temp_dir("test_from_file"); let test_file = format!("{}/test.toml", dir.path().to_str().unwrap()); let s = toml::to_string(&config).unwrap(); diff --git a/src/cmd/tests/cli.rs b/src/cmd/tests/cli.rs index 905191e13b..2b35c1c209 100644 --- a/src/cmd/tests/cli.rs +++ b/src/cmd/tests/cli.rs @@ -18,8 +18,8 @@ mod tests { use std::process::{Command, Stdio}; use std::time::Duration; + use common_test_util::temp_dir::create_temp_dir; use rexpect::session::PtyReplSession; - use tempdir::TempDir; struct Repl { repl: PtyReplSession, @@ -48,8 +48,8 @@ mod tests { #[test] fn test_repl() { - let data_dir = TempDir::new_in("/tmp", "data").unwrap(); - let wal_dir = TempDir::new_in("/tmp", "wal").unwrap(); + let data_dir = create_temp_dir("data"); + let wal_dir = create_temp_dir("wal"); let mut bin_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); bin_path.push("../../target/debug"); diff --git a/src/common/catalog/Cargo.toml b/src/common/catalog/Cargo.toml index 2ed50f13a0..57e4acb684 100644 --- a/src/common/catalog/Cargo.toml +++ b/src/common/catalog/Cargo.toml @@ -17,5 +17,4 @@ snafu = { version = "0.7", features = ["backtraces"] } [dev-dependencies] chrono.workspace = true -tempdir = "0.3" tokio.workspace = true diff --git a/src/common/procedure/Cargo.toml b/src/common/procedure/Cargo.toml index 83e601df61..2d6ce54118 100644 --- a/src/common/procedure/Cargo.toml +++ b/src/common/procedure/Cargo.toml @@ -19,5 +19,5 @@ tokio.workspace = true uuid.workspace = true [dev-dependencies] +common-test-util = { path = "../test-util" } futures-util.workspace = true -tempdir = "0.3" diff --git a/src/common/procedure/src/local.rs b/src/common/procedure/src/local.rs index aa74032303..36a97b6e3e 100644 --- a/src/common/procedure/src/local.rs +++ b/src/common/procedure/src/local.rs @@ -409,9 +409,9 @@ impl ProcedureManager for LocalManager { /// Create a new [ProcedureMeta] for test purpose. #[cfg(test)] mod test_util { + use common_test_util::temp_dir::TempDir; use object_store::services::Fs as Builder; use object_store::ObjectStoreBuilder; - use tempdir::TempDir; use super::*; @@ -430,7 +430,7 @@ mod test_util { mod tests { use common_error::mock::MockError; use common_error::prelude::StatusCode; - use tempdir::TempDir; + use common_test_util::temp_dir::create_temp_dir; use super::*; use crate::error::Error; @@ -540,7 +540,7 @@ mod tests { #[test] fn test_register_loader() { - let dir = TempDir::new("register").unwrap(); + let dir = create_temp_dir("register"); let config = ManagerConfig { object_store: test_util::new_object_store(&dir), }; @@ -558,7 +558,7 @@ mod tests { #[tokio::test] async fn test_recover() { - let dir = TempDir::new("recover").unwrap(); + let dir = create_temp_dir("recover"); let object_store = test_util::new_object_store(&dir); let config = ManagerConfig { object_store: object_store.clone(), @@ -603,7 +603,7 @@ mod tests { #[tokio::test] async fn test_submit_procedure() { - let dir = TempDir::new("submit").unwrap(); + let dir = create_temp_dir("submit"); let config = ManagerConfig { object_store: test_util::new_object_store(&dir), }; @@ -649,7 +649,7 @@ mod tests { #[tokio::test] async fn test_state_changed_on_err() { - let dir = TempDir::new("on_err").unwrap(); + let dir = create_temp_dir("on_err"); let config = ManagerConfig { object_store: test_util::new_object_store(&dir), }; diff --git a/src/common/procedure/src/local/runner.rs b/src/common/procedure/src/local/runner.rs index 46d2effa13..1716f20d74 100644 --- a/src/common/procedure/src/local/runner.rs +++ b/src/common/procedure/src/local/runner.rs @@ -395,10 +395,10 @@ mod tests { use common_error::ext::PlainError; use common_error::mock::MockError; use common_error::prelude::StatusCode; + use common_test_util::temp_dir::create_temp_dir; use futures_util::future::BoxFuture; use futures_util::{FutureExt, TryStreamExt}; use object_store::ObjectStore; - use tempdir::TempDir; use super::*; use crate::local::test_util; @@ -511,7 +511,7 @@ mod tests { exec_fn, }; - let dir = TempDir::new("normal").unwrap(); + let dir = create_temp_dir("normal"); let meta = normal.new_meta(ROOT_ID); let ctx = context_without_provider(meta.id); let object_store = test_util::new_object_store(&dir); @@ -559,7 +559,7 @@ mod tests { exec_fn, }; - let dir = TempDir::new("suspend").unwrap(); + let dir = create_temp_dir("suspend"); let meta = suspend.new_meta(ROOT_ID); let ctx = context_without_provider(meta.id); let object_store = test_util::new_object_store(&dir); @@ -658,7 +658,7 @@ mod tests { exec_fn, }; - let dir = TempDir::new("parent").unwrap(); + let dir = create_temp_dir("parent"); let meta = parent.new_meta(ROOT_ID); let procedure_id = meta.id; @@ -700,7 +700,7 @@ mod tests { exec_fn, }; - let dir = TempDir::new("fail").unwrap(); + let dir = create_temp_dir("fail"); let meta = fail.new_meta(ROOT_ID); let ctx = context_without_provider(meta.id); let object_store = test_util::new_object_store(&dir); @@ -735,7 +735,7 @@ mod tests { exec_fn, }; - let dir = TempDir::new("retry_later").unwrap(); + let dir = create_temp_dir("retry_later"); let meta = retry_later.new_meta(ROOT_ID); let ctx = context_without_provider(meta.id); let object_store = test_util::new_object_store(&dir); @@ -806,7 +806,7 @@ mod tests { exec_fn, }; - let dir = TempDir::new("child_err").unwrap(); + let dir = create_temp_dir("child_err"); let meta = parent.new_meta(ROOT_ID); let object_store = test_util::new_object_store(&dir); diff --git a/src/common/procedure/src/store.rs b/src/common/procedure/src/store.rs index 5198e1c110..fff460ed7a 100644 --- a/src/common/procedure/src/store.rs +++ b/src/common/procedure/src/store.rs @@ -246,9 +246,9 @@ impl ParsedKey { #[cfg(test)] mod tests { use async_trait::async_trait; + use common_test_util::temp_dir::{create_temp_dir, TempDir}; use object_store::services::Fs as Builder; use object_store::ObjectStoreBuilder; - use tempdir::TempDir; use super::*; use crate::{Context, LockKey, Procedure, Status}; @@ -373,7 +373,7 @@ mod tests { #[tokio::test] async fn test_store_procedure() { - let dir = TempDir::new("store_procedure").unwrap(); + let dir = create_temp_dir("store_procedure"); let store = procedure_store_for_test(&dir); let procedure_id = ProcedureId::random(); @@ -398,7 +398,7 @@ mod tests { #[tokio::test] async fn test_commit_procedure() { - let dir = TempDir::new("commit_procedure").unwrap(); + let dir = create_temp_dir("commit_procedure"); let store = procedure_store_for_test(&dir); let procedure_id = ProcedureId::random(); @@ -416,7 +416,7 @@ mod tests { #[tokio::test] async fn test_rollback_procedure() { - let dir = TempDir::new("rollback_procedure").unwrap(); + let dir = create_temp_dir("rollback_procedure"); let store = procedure_store_for_test(&dir); let procedure_id = ProcedureId::random(); @@ -434,7 +434,7 @@ mod tests { #[tokio::test] async fn test_load_messages() { - let dir = TempDir::new("load_messages").unwrap(); + let dir = create_temp_dir("load_messages"); let store = procedure_store_for_test(&dir); // store 3 steps diff --git a/src/common/procedure/src/store/state_store.rs b/src/common/procedure/src/store/state_store.rs index 97cebdbd5a..baefb3db64 100644 --- a/src/common/procedure/src/store/state_store.rs +++ b/src/common/procedure/src/store/state_store.rs @@ -115,15 +115,15 @@ impl StateStore for ObjectStateStore { #[cfg(test)] mod tests { + use common_test_util::temp_dir::create_temp_dir; use object_store::services::Fs as Builder; use object_store::ObjectStoreBuilder; - use tempdir::TempDir; use super::*; #[tokio::test] async fn test_object_state_store() { - let dir = TempDir::new("state_store").unwrap(); + let dir = create_temp_dir("state_store"); let store_dir = dir.path().to_str().unwrap(); let accessor = Builder::default().root(store_dir).build().unwrap(); let object_store = ObjectStore::new(accessor).finish(); diff --git a/src/common/test-util/Cargo.toml b/src/common/test-util/Cargo.toml new file mode 100644 index 0000000000..ca541cde5e --- /dev/null +++ b/src/common/test-util/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "common-test-util" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +tempfile.workspace = true diff --git a/src/common/test-util/src/lib.rs b/src/common/test-util/src/lib.rs new file mode 100644 index 0000000000..b3e734870a --- /dev/null +++ b/src/common/test-util/src/lib.rs @@ -0,0 +1,15 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod temp_dir; diff --git a/src/common/test-util/src/temp_dir.rs b/src/common/test-util/src/temp_dir.rs new file mode 100644 index 0000000000..407da333dc --- /dev/null +++ b/src/common/test-util/src/temp_dir.rs @@ -0,0 +1,23 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub use tempfile::{NamedTempFile, TempDir}; + +pub fn create_temp_dir(prefix: &str) -> TempDir { + tempfile::Builder::new().prefix(prefix).tempdir().unwrap() +} + +pub fn create_named_temp_file() -> NamedTempFile { + NamedTempFile::new().unwrap() +} diff --git a/src/datanode/Cargo.toml b/src/datanode/Cargo.toml index af37f9a8f3..af2d26cffb 100644 --- a/src/datanode/Cargo.toml +++ b/src/datanode/Cargo.toml @@ -69,7 +69,7 @@ url = "2.3.1" [dev-dependencies] axum-test-helper = { git = "https://github.com/sunng87/axum-test-helper.git", branch = "patch-1" } client = { path = "../client" } +common-test-util = { path = "../common/test-util" } common-query = { path = "../common/query" } datafusion-common.workspace = true -tempdir = "0.3" toml = "0.5" diff --git a/src/datanode/src/sql.rs b/src/datanode/src/sql.rs index c0a9f5f4e0..aa7632ab8b 100644 --- a/src/datanode/src/sql.rs +++ b/src/datanode/src/sql.rs @@ -150,6 +150,7 @@ mod tests { use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; use common_query::logical_plan::Expr; use common_query::physical_plan::PhysicalPlanRef; + use common_test_util::temp_dir::create_temp_dir; use common_time::timestamp::Timestamp; use datatypes::prelude::ConcreteDataType; use datatypes::schema::{ColumnSchema, SchemaBuilder, SchemaRef}; @@ -170,7 +171,6 @@ mod tests { use table::error::Result as TableResult; use table::metadata::TableInfoRef; use table::Table; - use tempdir::TempDir; use super::*; use crate::error::Error; @@ -221,7 +221,7 @@ mod tests { #[tokio::test] async fn test_statement_to_request() { - let dir = TempDir::new("setup_test_engine_and_table").unwrap(); + let dir = create_temp_dir("setup_test_engine_and_table"); let store_dir = dir.path().to_string_lossy(); let accessor = Builder::default().root(&store_dir).build().unwrap(); let object_store = ObjectStore::new(accessor).finish(); diff --git a/src/datanode/src/tests/test_util.rs b/src/datanode/src/tests/test_util.rs index d544ae682c..28591feaf1 100644 --- a/src/datanode/src/tests/test_util.rs +++ b/src/datanode/src/tests/test_util.rs @@ -17,6 +17,7 @@ use std::sync::Arc; use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, MIN_USER_TABLE_ID}; use common_query::Output; use common_recordbatch::util; +use common_test_util::temp_dir::{create_temp_dir, TempDir}; use datatypes::data_type::ConcreteDataType; use datatypes::schema::{ColumnSchema, RawSchema}; use mito::config::EngineConfig; @@ -26,7 +27,6 @@ use servers::Mode; use snafu::ResultExt; use table::engine::{EngineContext, TableEngineRef}; use table::requests::{CreateTableRequest, TableOptions}; -use tempdir::TempDir; use crate::datanode::{DatanodeOptions, FileConfig, ObjectStoreConfig, ProcedureConfig, WalConfig}; use crate::error::{CreateTableSnafu, Result}; @@ -55,7 +55,7 @@ impl MockInstance { pub(crate) async fn with_procedure_enabled(name: &str) -> Self { let (mut opts, _guard) = create_tmp_dir_and_datanode_opts(name); - let procedure_dir = TempDir::new(&format!("gt_procedure_{name}")).unwrap(); + let procedure_dir = create_temp_dir(&format!("gt_procedure_{name}")); opts.procedure = Some(ProcedureConfig { store: ObjectStoreConfig::File(FileConfig { data_dir: procedure_dir.path().to_str().unwrap().to_string(), @@ -87,8 +87,8 @@ struct TestGuard { } fn create_tmp_dir_and_datanode_opts(name: &str) -> (DatanodeOptions, TestGuard) { - let wal_tmp_dir = TempDir::new(&format!("gt_wal_{name}")).unwrap(); - let data_tmp_dir = TempDir::new(&format!("gt_data_{name}")).unwrap(); + let wal_tmp_dir = create_temp_dir(&format!("gt_wal_{name}")); + let data_tmp_dir = create_temp_dir(&format!("gt_data_{name}")); let opts = DatanodeOptions { wal: WalConfig { dir: wal_tmp_dir.path().to_str().unwrap().to_string(), diff --git a/src/frontend/Cargo.toml b/src/frontend/Cargo.toml index 53bb74eb35..401b84c9c2 100644 --- a/src/frontend/Cargo.toml +++ b/src/frontend/Cargo.toml @@ -48,10 +48,10 @@ tokio.workspace = true tonic.workspace = true [dev-dependencies] +common-test-util = { path = "../common/test-util" } datanode = { path = "../datanode" } futures = "0.3" meta-srv = { path = "../meta-srv", features = ["mock"] } strfmt = "0.2" -tempdir = "0.3" toml = "0.5" tower = "0.4" diff --git a/src/frontend/src/tests.rs b/src/frontend/src/tests.rs index 91fb322528..d9e1a8e9b5 100644 --- a/src/frontend/src/tests.rs +++ b/src/frontend/src/tests.rs @@ -20,6 +20,7 @@ use catalog::remote::MetaKvBackend; use client::Client; use common_grpc::channel_manager::ChannelManager; use common_runtime::Builder as RuntimeBuilder; +use common_test_util::temp_dir::{create_temp_dir, TempDir}; use datanode::datanode::{DatanodeOptions, FileConfig, ObjectStoreConfig, WalConfig}; use datanode::instance::Instance as DatanodeInstance; use meta_client::client::MetaClientBuilder; @@ -33,7 +34,6 @@ use partition::route::TableRoutes; use servers::grpc::GrpcServer; use servers::query_handler::grpc::ServerGrpcQueryHandlerAdaptor; use servers::Mode; -use tempdir::TempDir; use tonic::transport::Server; use tower::service_fn; @@ -75,8 +75,8 @@ pub(crate) async fn create_standalone_instance(test_name: &str) -> MockStandalon } fn create_tmp_dir_and_datanode_opts(name: &str) -> (DatanodeOptions, TestGuard) { - let wal_tmp_dir = TempDir::new(&format!("gt_wal_{name}")).unwrap(); - let data_tmp_dir = TempDir::new(&format!("gt_data_{name}")).unwrap(); + let wal_tmp_dir = create_temp_dir(&format!("gt_wal_{name}")); + let data_tmp_dir = create_temp_dir(&format!("gt_data_{name}")); let opts = DatanodeOptions { wal: WalConfig { dir: wal_tmp_dir.path().to_str().unwrap().to_string(), @@ -161,8 +161,8 @@ async fn create_distributed_datanode( datanode_id: u64, meta_srv: MockInfo, ) -> (Arc, TestGuard) { - let wal_tmp_dir = TempDir::new(&format!("gt_wal_{test_name}_dist_dn_{datanode_id}")).unwrap(); - let data_tmp_dir = TempDir::new(&format!("gt_data_{test_name}_dist_dn_{datanode_id}")).unwrap(); + let wal_tmp_dir = create_temp_dir(&format!("gt_wal_{test_name}_dist_dn_{datanode_id}")); + let data_tmp_dir = create_temp_dir(&format!("gt_data_{test_name}_dist_dn_{datanode_id}")); let opts = DatanodeOptions { node_id: Some(datanode_id), wal: WalConfig { diff --git a/src/log-store/Cargo.toml b/src/log-store/Cargo.toml index f7a83e5a5b..f2a25afab2 100644 --- a/src/log-store/Cargo.toml +++ b/src/log-store/Cargo.toml @@ -28,9 +28,9 @@ protobuf = { version = "2", features = ["bytes"] } raft-engine = "0.3" snafu = { version = "0.7", features = ["backtraces"] } store-api = { path = "../store-api" } -tempdir = "0.3" tokio.workspace = true tokio-util.workspace = true [dev-dependencies] +common-test-util = { path = "../common/test-util" } rand = "0.8" diff --git a/src/log-store/src/raft_engine/log_store.rs b/src/log-store/src/raft_engine/log_store.rs index e56a560725..48dfc187b2 100644 --- a/src/log-store/src/raft_engine/log_store.rs +++ b/src/log-store/src/raft_engine/log_store.rs @@ -338,12 +338,12 @@ mod tests { use std::time::Duration; use common_telemetry::debug; + use common_test_util::temp_dir::create_temp_dir; use futures_util::StreamExt; use raft_engine::ReadableSize; use store_api::logstore::entry_stream::SendableEntryStream; use store_api::logstore::namespace::Namespace as NamespaceTrait; use store_api::logstore::LogStore; - use tempdir::TempDir; use crate::config::LogConfig; use crate::error::Error; @@ -352,7 +352,7 @@ mod tests { #[tokio::test] async fn test_open_logstore() { - let dir = TempDir::new("raft-engine-logstore-test").unwrap(); + let dir = create_temp_dir("raft-engine-logstore-test"); let logstore = RaftEngineLogStore::try_new(LogConfig { log_file_dir: dir.path().to_str().unwrap().to_string(), ..Default::default() @@ -366,7 +366,7 @@ mod tests { #[tokio::test] async fn test_manage_namespace() { - let dir = TempDir::new("raft-engine-logstore-test").unwrap(); + let dir = create_temp_dir("raft-engine-logstore-test"); let mut logstore = RaftEngineLogStore::try_new(LogConfig { log_file_dir: dir.path().to_str().unwrap().to_string(), ..Default::default() @@ -393,7 +393,7 @@ mod tests { #[tokio::test] async fn test_append_and_read() { - let dir = TempDir::new("raft-engine-logstore-test").unwrap(); + let dir = create_temp_dir("raft-engine-logstore-test"); let logstore = RaftEngineLogStore::try_new(LogConfig { log_file_dir: dir.path().to_str().unwrap().to_string(), ..Default::default() @@ -434,7 +434,7 @@ mod tests { #[tokio::test] async fn test_reopen() { - let dir = TempDir::new("raft-engine-logstore-reopen-test").unwrap(); + let dir = create_temp_dir("raft-engine-logstore-reopen-test"); { let logstore = RaftEngineLogStore::try_new(LogConfig { log_file_dir: dir.path().to_str().unwrap().to_string(), @@ -491,7 +491,7 @@ mod tests { #[tokio::test] async fn test_compaction() { common_telemetry::init_default_ut_logging(); - let dir = TempDir::new("raft-engine-logstore-test").unwrap(); + let dir = create_temp_dir("raft-engine-logstore-test"); let config = LogConfig { log_file_dir: dir.path().to_str().unwrap().to_string(), @@ -524,7 +524,7 @@ mod tests { #[tokio::test] async fn test_obsolete() { common_telemetry::init_default_ut_logging(); - let dir = TempDir::new("raft-engine-logstore-test").unwrap(); + let dir = create_temp_dir("raft-engine-logstore-test"); let config = LogConfig { log_file_dir: dir.path().to_str().unwrap().to_string(), diff --git a/src/log-store/src/test_util/log_store_util.rs b/src/log-store/src/test_util/log_store_util.rs index 684f368afd..c1ec201f1d 100644 --- a/src/log-store/src/test_util/log_store_util.rs +++ b/src/log-store/src/test_util/log_store_util.rs @@ -12,21 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -use tempdir::TempDir; - use crate::raft_engine::log_store::RaftEngineLogStore; use crate::LogConfig; -/// Create a tmp directory for write log, used for test. -// TODO: Add a test feature -pub async fn create_tmp_local_file_log_store(dir: &str) -> (RaftEngineLogStore, TempDir) { - let dir = TempDir::new(dir).unwrap(); +/// Create a write log for the provided path, used for test. +pub async fn create_tmp_local_file_log_store(path: &str) -> RaftEngineLogStore { let cfg = LogConfig { file_size: 128 * 1024, - log_file_dir: dir.path().to_str().unwrap().to_string(), + log_file_dir: path.to_string(), ..Default::default() }; - - let logstore = RaftEngineLogStore::try_new(cfg).await.unwrap(); - (logstore, dir) + RaftEngineLogStore::try_new(cfg).await.unwrap() } diff --git a/src/mito/Cargo.toml b/src/mito/Cargo.toml index c59e265fbc..29285b4385 100644 --- a/src/mito/Cargo.toml +++ b/src/mito/Cargo.toml @@ -6,7 +6,7 @@ license.workspace = true [features] default = [] -test = ["tempdir"] +test = ["common-test-util"] [dependencies] anymap = "1.0.0-beta.2" @@ -33,8 +33,8 @@ snafu.workspace = true storage = { path = "../storage" } store-api = { path = "../store-api" } table = { path = "../table" } -tempdir = { version = "0.3", optional = true } +common-test-util = { path = "../common/test-util", optional = true } tokio.workspace = true [dev-dependencies] -tempdir = { version = "0.3" } +common-test-util = { path = "../common/test-util" } diff --git a/src/mito/src/engine/procedure.rs b/src/mito/src/engine/procedure.rs index f044329f27..2985f3c4cb 100644 --- a/src/mito/src/engine/procedure.rs +++ b/src/mito/src/engine/procedure.rs @@ -40,11 +40,11 @@ mod procedure_test_util { use common_procedure::{ BoxedProcedure, Context, ContextProvider, ProcedureId, ProcedureState, Result, Status, }; + use common_test_util::temp_dir::TempDir; use log_store::NoopLogStore; use storage::compaction::noop::NoopCompactionScheduler; use storage::config::EngineConfig as StorageEngineConfig; use storage::EngineImpl; - use tempdir::TempDir; use super::*; use crate::engine::{EngineConfig, MitoEngine}; diff --git a/src/mito/src/engine/tests.rs b/src/mito/src/engine/tests.rs index 0758f36b83..d533a01621 100644 --- a/src/mito/src/engine/tests.rs +++ b/src/mito/src/engine/tests.rs @@ -17,6 +17,7 @@ use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; use common_query::physical_plan::SessionContext; use common_recordbatch::util; +use common_test_util::temp_dir::TempDir; use datatypes::prelude::ConcreteDataType; use datatypes::schema::{ColumnDefaultConstraint, ColumnSchema, RawSchema}; use datatypes::value::Value; @@ -31,7 +32,6 @@ use storage::EngineImpl; use store_api::manifest::Manifest; use store_api::storage::ReadContext; use table::requests::{AddColumnRequest, AlterKind, DeleteRequest, TableOptions}; -use tempdir::TempDir; use super::*; use crate::table::test_util; diff --git a/src/mito/src/table/test_util.rs b/src/mito/src/table/test_util.rs index 78eba284c3..be637cc498 100644 --- a/src/mito/src/table/test_util.rs +++ b/src/mito/src/table/test_util.rs @@ -17,6 +17,7 @@ use std::collections::HashMap; use std::sync::Arc; use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; +use common_test_util::temp_dir::{create_temp_dir, TempDir}; use datatypes::prelude::ConcreteDataType; use datatypes::schema::{ColumnSchema, RawSchema, Schema, SchemaBuilder, SchemaRef}; use datatypes::vectors::VectorRef; @@ -30,7 +31,6 @@ use table::engine::{EngineContext, TableEngine}; use table::metadata::{TableInfo, TableInfoBuilder, TableMetaBuilder, TableType}; use table::requests::{CreateTableRequest, InsertRequest, TableOptions}; use table::TableRef; -use tempdir::TempDir; use crate::config::EngineConfig; use crate::engine::{MitoEngine, MITO_ENGINE}; @@ -96,7 +96,7 @@ pub fn build_test_table_info() -> TableInfo { } pub async fn new_test_object_store(prefix: &str) -> (TempDir, ObjectStore) { - let dir = TempDir::new(prefix).unwrap(); + let dir = create_temp_dir(prefix); let store_dir = dir.path().to_string_lossy(); let accessor = Builder::default().root(&store_dir).build().unwrap(); (dir, ObjectStore::new(accessor).finish()) diff --git a/src/object-store/Cargo.toml b/src/object-store/Cargo.toml index 9390492e03..e664c5842a 100644 --- a/src/object-store/Cargo.toml +++ b/src/object-store/Cargo.toml @@ -14,5 +14,5 @@ tokio.workspace = true [dev-dependencies] anyhow = "1.0" common-telemetry = { path = "../common/telemetry" } -tempdir = "0.3" +common-test-util = { path = "../common/test-util" } uuid.workspace = true diff --git a/src/object-store/tests/object_store_test.rs b/src/object-store/tests/object_store_test.rs index 81d4dd5ec6..4154506610 100644 --- a/src/object-store/tests/object_store_test.rs +++ b/src/object-store/tests/object_store_test.rs @@ -17,13 +17,13 @@ use std::sync::Arc; use anyhow::Result; use common_telemetry::logging; +use common_test_util::temp_dir::create_temp_dir; use object_store::cache_policy::LruCacheLayer; use object_store::services::{Fs, S3}; use object_store::test_util::TempFolder; use object_store::{util, Object, ObjectLister, ObjectMode, ObjectStore, ObjectStoreBuilder}; use opendal::services::Oss; use opendal::Operator; -use tempdir::TempDir; async fn test_object_crud(store: &ObjectStore) -> Result<()> { // Create object handler. @@ -93,8 +93,8 @@ async fn test_object_list(store: &ObjectStore) -> Result<()> { #[tokio::test] async fn test_fs_backend() -> Result<()> { - let data_dir = TempDir::new("test_fs_backend")?; - let tmp_dir = TempDir::new("test_fs_backend")?; + let data_dir = create_temp_dir("test_fs_backend"); + let tmp_dir = create_temp_dir("test_fs_backend"); let store = ObjectStore::new( Fs::default() .root(&data_dir.path().to_string_lossy()) @@ -195,7 +195,7 @@ async fn assert_cache_files( #[tokio::test] async fn test_object_store_cache_policy() -> Result<()> { // create file storage - let root_dir = TempDir::new("test_fs_backend")?; + let root_dir = create_temp_dir("test_fs_backend"); let store = ObjectStore::new( Fs::default() .root(&root_dir.path().to_string_lossy()) @@ -204,7 +204,7 @@ async fn test_object_store_cache_policy() -> Result<()> { ); // create file cache layer - let cache_dir = TempDir::new("test_fs_cache")?; + let cache_dir = create_temp_dir("test_fs_cache"); let cache_acc = Fs::default() .root(&cache_dir.path().to_string_lossy()) .atomic_write_dir(&cache_dir.path().to_string_lossy()) diff --git a/src/script/Cargo.toml b/src/script/Cargo.toml index 4f48214669..e7f6f1efe5 100644 --- a/src/script/Cargo.toml +++ b/src/script/Cargo.toml @@ -68,11 +68,11 @@ table = { path = "../table" } tokio.workspace = true [dev-dependencies] +common-test-util = { path = "../common/test-util" } log-store = { path = "../log-store" } mito = { path = "../mito", features = ["test"] } ron = "0.7" serde = { version = "1.0", features = ["derive"] } storage = { path = "../storage" } store-api = { path = "../store-api" } -tempdir = "0.3" tokio-test = "0.4" diff --git a/src/script/src/manager.rs b/src/script/src/manager.rs index ba79a361c8..dc8125221b 100644 --- a/src/script/src/manager.rs +++ b/src/script/src/manager.rs @@ -120,17 +120,18 @@ mod tests { use super::*; type DefaultEngine = MitoEngine>; + + use common_test_util::temp_dir::create_temp_dir; use log_store::raft_engine::log_store::RaftEngineLogStore; use log_store::LogConfig; use mito::engine::MitoEngine; use storage::compaction::noop::NoopCompactionScheduler; use storage::config::EngineConfig as StorageEngineConfig; use storage::EngineImpl; - use tempdir::TempDir; #[tokio::test] async fn test_insert_find_compile_script() { - let wal_dir = TempDir::new("test_insert_find_compile_script_wal").unwrap(); + let wal_dir = create_temp_dir("test_insert_find_compile_script_wal"); let wal_dir_str = wal_dir.path().to_string_lossy(); common_telemetry::init_default_ut_logging(); diff --git a/src/servers/Cargo.toml b/src/servers/Cargo.toml index b8203dfd1c..2e0ba83b53 100644 --- a/src/servers/Cargo.toml +++ b/src/servers/Cargo.toml @@ -76,6 +76,7 @@ tower-http = { version = "0.3", features = ["full"] } axum-test-helper = { git = "https://github.com/sunng87/axum-test-helper.git", branch = "patch-1" } client = { path = "../client" } common-base = { path = "../common/base" } +common-test-util = { path = "../common/test-util" } mysql_async = { version = "0.31", default-features = false, features = [ "default-rustls", ] } @@ -83,7 +84,6 @@ rand = "0.8" script = { path = "../script", features = ["python"] } serde_json = "1.0" table = { path = "../table" } -tempdir = "0.3" tokio-postgres = "0.7" tokio-postgres-rustls = "0.9" tokio-test = "0.4" diff --git a/src/servers/src/auth/user_provider.rs b/src/servers/src/auth/user_provider.rs index f425726b49..8fb6065dc3 100644 --- a/src/servers/src/auth/user_provider.rs +++ b/src/servers/src/auth/user_provider.rs @@ -187,8 +187,8 @@ pub mod test { use std::fs::File; use std::io::{LineWriter, Write}; + use common_test_util::temp_dir::create_temp_dir; use session::context::UserInfo; - use tempdir::TempDir; use crate::auth::user_provider::{double_sha1, sha1_one, sha1_two, StaticUserProvider}; use crate::auth::{Identity, Password, UserProvider}; @@ -245,7 +245,7 @@ pub mod test { #[tokio::test] async fn test_file_provider() { - let dir = TempDir::new("test_file_provider").unwrap(); + let dir = create_temp_dir("test_file_provider"); let file_path = format!("{}/test_file_provider", dir.path().to_str().unwrap()); { // write a tmp file diff --git a/src/storage/Cargo.toml b/src/storage/Cargo.toml index 0da31bafa3..48d141075e 100644 --- a/src/storage/Cargo.toml +++ b/src/storage/Cargo.toml @@ -45,10 +45,10 @@ uuid.workspace = true [dev-dependencies] atomic_float = "0.1" criterion = "0.3" +common-test-util = { path = "../common/test-util" } datatypes = { path = "../datatypes", features = ["test"] } log-store = { path = "../log-store" } rand = "0.8" -tempdir = "0.3" [build-dependencies] tonic-build = "0.8" diff --git a/src/storage/src/compaction/writer.rs b/src/storage/src/compaction/writer.rs index b09d2184e5..71ea9b39f0 100644 --- a/src/storage/src/compaction/writer.rs +++ b/src/storage/src/compaction/writer.rs @@ -85,6 +85,7 @@ mod tests { use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; + use common_test_util::temp_dir::create_temp_dir; use common_time::Timestamp; use datatypes::prelude::{LogicalTypeId, ScalarVector, ScalarVectorBuilder}; use datatypes::timestamp::TimestampMillisecond; @@ -94,7 +95,6 @@ mod tests { use object_store::services::Fs; use object_store::{ObjectStore, ObjectStoreBuilder}; use store_api::storage::{ChunkReader, OpType, SequenceNumber}; - use tempdir::TempDir; use super::*; use crate::file_purger::noop::new_noop_file_purger; @@ -274,7 +274,7 @@ mod tests { #[tokio::test] async fn test_sst_reader() { - let dir = TempDir::new("write_parquet").unwrap(); + let dir = create_temp_dir("write_parquet"); let path = dir.path().to_str().unwrap(); let backend = Fs::default().root(path).build().unwrap(); let object_store = ObjectStore::new(backend).finish(); @@ -351,7 +351,7 @@ mod tests { /// and check the output contains the same data as input files. #[tokio::test] async fn test_sst_split() { - let dir = TempDir::new("write_parquet").unwrap(); + let dir = create_temp_dir("write_parquet"); let path = dir.path().to_str().unwrap(); let backend = Fs::default().root(path).build().unwrap(); let object_store = ObjectStore::new(backend).finish(); diff --git a/src/storage/src/engine.rs b/src/storage/src/engine.rs index 5122c8e918..24fc19d2f6 100644 --- a/src/storage/src/engine.rs +++ b/src/storage/src/engine.rs @@ -377,12 +377,12 @@ impl EngineInner { #[cfg(test)] mod tests { + use common_test_util::temp_dir::create_temp_dir; use datatypes::type_id::LogicalTypeId; use log_store::test_util::log_store_util; use object_store::services::Fs; use object_store::ObjectStoreBuilder; use store_api::storage::Region; - use tempdir::TempDir; use super::*; use crate::compaction::noop::NoopCompactionScheduler; @@ -390,9 +390,10 @@ mod tests { #[tokio::test] async fn test_create_new_region() { - let (log_store, _tmp) = - log_store_util::create_tmp_local_file_log_store("test_engine_wal").await; - let dir = TempDir::new("test_create_new_region").unwrap(); + let log_file_dir = create_temp_dir("test_engine_wal"); + let log_file_dir_path = log_file_dir.path().to_str().unwrap(); + let log_store = log_store_util::create_tmp_local_file_log_store(log_file_dir_path).await; + let dir = create_temp_dir("test_create_new_region"); let store_dir = dir.path().to_string_lossy(); let accessor = Fs::default().root(&store_dir).build().unwrap(); diff --git a/src/storage/src/file_purger.rs b/src/storage/src/file_purger.rs index f9a9840197..e2fb77d9fb 100644 --- a/src/storage/src/file_purger.rs +++ b/src/storage/src/file_purger.rs @@ -106,10 +106,10 @@ pub mod noop { #[cfg(test)] mod tests { + use common_test_util::temp_dir::create_temp_dir; use object_store::services::Fs; use object_store::{ObjectStore, ObjectStoreBuilder}; use store_api::storage::OpType; - use tempdir::TempDir; use super::*; use crate::file_purger::noop::NoopFilePurgeHandler; @@ -167,7 +167,7 @@ mod tests { #[tokio::test] async fn test_file_purger_handler() { - let dir = TempDir::new("file-purge").unwrap(); + let dir = create_temp_dir("file-purge"); let object_store = ObjectStore::new( Fs::default() .root(dir.path().to_str().unwrap()) @@ -205,7 +205,7 @@ mod tests { #[tokio::test] async fn test_file_purge_loop() { common_telemetry::init_default_ut_logging(); - let dir = TempDir::new("file-purge").unwrap(); + let dir = create_temp_dir("file-purge"); let object_store = ObjectStore::new( Fs::default() .root(dir.path().to_str().unwrap()) diff --git a/src/storage/src/manifest/region.rs b/src/storage/src/manifest/region.rs index e6e5678c2d..ec82ac53bc 100644 --- a/src/storage/src/manifest/region.rs +++ b/src/storage/src/manifest/region.rs @@ -22,11 +22,11 @@ pub type RegionManifest = ManifestImpl; mod tests { use std::sync::Arc; + use common_test_util::temp_dir::create_temp_dir; use object_store::services::Fs; use object_store::{ObjectStore, ObjectStoreBuilder}; use store_api::manifest::action::ProtocolAction; use store_api::manifest::{Manifest, MetaActionIterator, MAX_VERSION}; - use tempdir::TempDir; use super::*; use crate::manifest::test_utils::*; @@ -35,7 +35,7 @@ mod tests { #[tokio::test] async fn test_region_manifest() { common_telemetry::init_default_ut_logging(); - let tmp_dir = TempDir::new("test_region_manifest").unwrap(); + let tmp_dir = create_temp_dir("test_region_manifest"); let object_store = ObjectStore::new( Fs::default() .root(&tmp_dir.path().to_string_lossy()) diff --git a/src/storage/src/manifest/storage.rs b/src/storage/src/manifest/storage.rs index 6061dba617..a55b3fb925 100644 --- a/src/storage/src/manifest/storage.rs +++ b/src/storage/src/manifest/storage.rs @@ -277,16 +277,16 @@ impl ManifestLogStorage for ManifestObjectStore { #[cfg(test)] mod tests { + use common_test_util::temp_dir::create_temp_dir; use object_store::services::Fs; use object_store::{ObjectStore, ObjectStoreBuilder}; - use tempdir::TempDir; use super::*; #[tokio::test] async fn test_manifest_log_store() { common_telemetry::init_default_ut_logging(); - let tmp_dir = TempDir::new("test_manifest_log_store").unwrap(); + let tmp_dir = create_temp_dir("test_manifest_log_store"); let object_store = ObjectStore::new( Fs::default() .root(&tmp_dir.path().to_string_lossy()) diff --git a/src/storage/src/region/tests.rs b/src/storage/src/region/tests.rs index adf6c9c8ac..ae761b3155 100644 --- a/src/storage/src/region/tests.rs +++ b/src/storage/src/region/tests.rs @@ -23,6 +23,7 @@ mod projection; use std::collections::{HashMap, HashSet}; use common_telemetry::logging; +use common_test_util::temp_dir::create_temp_dir; use datatypes::prelude::{ScalarVector, WrapperType}; use datatypes::timestamp::TimestampMillisecond; use datatypes::type_id::LogicalTypeId; @@ -34,7 +35,6 @@ use object_store::{ObjectStore, ObjectStoreBuilder}; use store_api::storage::{ consts, Chunk, ChunkReader, RegionMeta, ScanRequest, SequenceNumber, Snapshot, WriteRequest, }; -use tempdir::TempDir; use super::*; use crate::file_purger::noop::NoopFilePurgeHandler; @@ -242,13 +242,10 @@ async fn test_new_region() { .build(); let metadata: RegionMetadata = desc.try_into().unwrap(); - let store_dir = TempDir::new("test_new_region") - .unwrap() - .path() - .to_string_lossy() - .to_string(); + let dir = create_temp_dir("test_new_region"); + let store_dir = dir.path().to_str().unwrap(); - let store_config = config_util::new_store_config(region_name, &store_dir).await; + let store_config = config_util::new_store_config(region_name, store_dir).await; let placeholder_memtable = store_config .memtable_builder .build(metadata.schema().clone()); @@ -278,7 +275,7 @@ async fn test_new_region() { #[tokio::test] async fn test_recover_region_manifets() { - let tmp_dir = TempDir::new("test_new_region").unwrap(); + let tmp_dir = create_temp_dir("test_new_region"); let memtable_builder = Arc::new(DefaultMemtableBuilder::default()) as _; let object_store = ObjectStore::new( diff --git a/src/storage/src/region/tests/alter.rs b/src/storage/src/region/tests/alter.rs index 2894ca4f19..1bdf429d12 100644 --- a/src/storage/src/region/tests/alter.rs +++ b/src/storage/src/region/tests/alter.rs @@ -15,6 +15,7 @@ use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; +use common_test_util::temp_dir::create_temp_dir; use datatypes::prelude::*; use datatypes::timestamp::TimestampMillisecond; use datatypes::vectors::{Int64Vector, TimestampMillisecondVector, VectorRef}; @@ -24,7 +25,6 @@ use store_api::storage::{ ColumnDescriptorBuilder, ColumnId, Region, RegionMeta, ScanRequest, SchemaRef, Snapshot, WriteRequest, WriteResponse, }; -use tempdir::TempDir; use crate::region::tests::{self, FileTesterBase}; use crate::region::{OpenOptions, RawRegionMetadata, RegionImpl, RegionMetadata}; @@ -266,7 +266,7 @@ fn check_schema_names(schema: &SchemaRef, names: &[&str]) { async fn test_alter_region_with_reopen() { common_telemetry::init_default_ut_logging(); - let dir = TempDir::new("alter-region").unwrap(); + let dir = create_temp_dir("alter-region"); let store_dir = dir.path().to_str().unwrap(); let mut tester = AlterTester::new(store_dir).await; @@ -338,7 +338,7 @@ async fn test_alter_region_with_reopen() { #[tokio::test] async fn test_alter_region() { - let dir = TempDir::new("alter-region").unwrap(); + let dir = create_temp_dir("alter-region"); let store_dir = dir.path().to_str().unwrap(); let tester = AlterTester::new(store_dir).await; @@ -377,7 +377,7 @@ async fn test_alter_region() { #[tokio::test] async fn test_put_old_schema_after_alter() { - let dir = TempDir::new("put-old").unwrap(); + let dir = create_temp_dir("put-old"); let store_dir = dir.path().to_str().unwrap(); let tester = AlterTester::new(store_dir).await; @@ -415,7 +415,7 @@ async fn test_put_old_schema_after_alter() { #[tokio::test] async fn test_replay_metadata_after_open() { - let dir = TempDir::new("replay-metadata-after-open").unwrap(); + let dir = create_temp_dir("replay-metadata-after-open"); let store_dir = dir.path().to_str().unwrap(); let mut tester = AlterTester::new(store_dir).await; diff --git a/src/storage/src/region/tests/basic.rs b/src/storage/src/region/tests/basic.rs index 8365b10c9d..f820eb8beb 100644 --- a/src/storage/src/region/tests/basic.rs +++ b/src/storage/src/region/tests/basic.rs @@ -15,9 +15,9 @@ //! Region read/write tests. use common_telemetry::info; +use common_test_util::temp_dir::create_temp_dir; use log_store::raft_engine::log_store::RaftEngineLogStore; use store_api::storage::{OpenOptions, SequenceNumber, WriteResponse}; -use tempdir::TempDir; use crate::error::Result; use crate::region::tests::{self, FileTesterBase}; @@ -120,7 +120,7 @@ impl Tester { #[tokio::test] async fn test_simple_put_scan() { - let dir = TempDir::new("put-scan").unwrap(); + let dir = create_temp_dir("put-scan"); let store_dir = dir.path().to_str().unwrap(); let tester = Tester::new(REGION_NAME, store_dir).await; @@ -140,7 +140,7 @@ async fn test_simple_put_scan() { #[tokio::test] async fn test_sequence_increase() { - let dir = TempDir::new("sequence").unwrap(); + let dir = create_temp_dir("sequence"); let store_dir = dir.path().to_str().unwrap(); let tester = Tester::new(REGION_NAME, store_dir).await; @@ -157,7 +157,7 @@ async fn test_sequence_increase() { async fn test_reopen() { common_telemetry::logging::init_default_ut_logging(); - let dir = TempDir::new("reopen").unwrap(); + let dir = create_temp_dir("reopen"); let store_dir = dir.path().to_str().unwrap(); let mut tester = Tester::new(REGION_NAME, store_dir).await; @@ -184,7 +184,7 @@ async fn test_reopen() { #[tokio::test] async fn test_open_empty() { - let dir = TempDir::new("open-empty").unwrap(); + let dir = create_temp_dir("open-empty"); let store_dir = dir.path().to_str().unwrap(); let mut tester = Tester::empty(REGION_NAME, store_dir).await; @@ -194,7 +194,7 @@ async fn test_open_empty() { #[tokio::test] async fn test_scan_different_batch() { - let dir = TempDir::new("different-batch").unwrap(); + let dir = create_temp_dir("different-batch"); let store_dir = dir.path().to_str().unwrap(); let mut tester = Tester::new(REGION_NAME, store_dir).await; @@ -216,7 +216,7 @@ async fn test_scan_different_batch() { #[tokio::test] async fn test_put_delete_scan() { common_telemetry::init_default_ut_logging(); - let dir = TempDir::new("put-delete-scan").unwrap(); + let dir = create_temp_dir("put-delete-scan"); let store_dir = dir.path().to_str().unwrap(); let mut tester = Tester::new(REGION_NAME, store_dir).await; @@ -246,7 +246,7 @@ async fn test_put_delete_scan() { #[tokio::test] async fn test_put_delete_absent_key() { - let dir = TempDir::new("put-delete-scan").unwrap(); + let dir = create_temp_dir("put-delete-scan"); let store_dir = dir.path().to_str().unwrap(); let mut tester = Tester::new(REGION_NAME, store_dir).await; diff --git a/src/storage/src/region/tests/close.rs b/src/storage/src/region/tests/close.rs index b166813a7a..dbc07c44a7 100644 --- a/src/storage/src/region/tests/close.rs +++ b/src/storage/src/region/tests/close.rs @@ -16,9 +16,9 @@ use std::sync::Arc; +use common_test_util::temp_dir::create_temp_dir; use log_store::raft_engine::log_store::RaftEngineLogStore; use store_api::storage::{AlterOperation, AlterRequest, Region, RegionMeta, WriteResponse}; -use tempdir::TempDir; use crate::engine; use crate::error::Error; @@ -87,7 +87,7 @@ impl CloseTester { #[tokio::test] async fn test_close_basic() { common_telemetry::init_default_ut_logging(); - let dir = TempDir::new("close-basic").unwrap(); + let dir = create_temp_dir("close-basic"); let store_dir = dir.path().to_str().unwrap(); let flush_switch = Arc::new(FlushSwitch::default()); @@ -123,7 +123,7 @@ async fn test_close_basic() { #[tokio::test] async fn test_close_wait_flush_done() { common_telemetry::init_default_ut_logging(); - let dir = TempDir::new("close-basic").unwrap(); + let dir = create_temp_dir("close-basic"); let store_dir = dir.path().to_str().unwrap(); let flush_switch = Arc::new(FlushSwitch::default()); diff --git a/src/storage/src/region/tests/flush.rs b/src/storage/src/region/tests/flush.rs index f22aca90c5..0d717272bb 100644 --- a/src/storage/src/region/tests/flush.rs +++ b/src/storage/src/region/tests/flush.rs @@ -16,9 +16,9 @@ use std::sync::Arc; +use common_test_util::temp_dir::create_temp_dir; use log_store::raft_engine::log_store::RaftEngineLogStore; use store_api::storage::{OpenOptions, WriteResponse}; -use tempdir::TempDir; use crate::engine; use crate::flush::FlushStrategyRef; @@ -100,7 +100,7 @@ impl FlushTester { async fn test_flush_and_stall() { common_telemetry::init_default_ut_logging(); - let dir = TempDir::new("flush-stall").unwrap(); + let dir = create_temp_dir("flush-stall"); let store_dir = dir.path().to_str().unwrap(); let flush_switch = Arc::new(FlushSwitch::default()); @@ -126,7 +126,7 @@ async fn test_flush_and_stall() { #[tokio::test] async fn test_flush_empty() { - let dir = TempDir::new("flush-empty").unwrap(); + let dir = create_temp_dir("flush-empty"); let store_dir = dir.path().to_str().unwrap(); let flush_switch = Arc::new(FlushSwitch::default()); @@ -159,7 +159,7 @@ async fn test_flush_empty() { async fn test_read_after_flush() { common_telemetry::init_default_ut_logging(); - let dir = TempDir::new("read-flush").unwrap(); + let dir = create_temp_dir("read-flush"); let store_dir = dir.path().to_str().unwrap(); let flush_switch = Arc::new(FlushSwitch::default()); @@ -192,7 +192,7 @@ async fn test_read_after_flush() { #[tokio::test] async fn test_merge_read_after_flush() { - let dir = TempDir::new("merge-read-flush").unwrap(); + let dir = create_temp_dir("merge-read-flush"); let store_dir = dir.path().to_str().unwrap(); let flush_switch = Arc::new(FlushSwitch::default()); diff --git a/src/storage/src/region/tests/projection.rs b/src/storage/src/region/tests/projection.rs index d1f5e1976d..79e840c554 100644 --- a/src/storage/src/region/tests/projection.rs +++ b/src/storage/src/region/tests/projection.rs @@ -15,6 +15,7 @@ use std::collections::HashMap; use std::sync::Arc; +use common_test_util::temp_dir::create_temp_dir; use datatypes::data_type::ConcreteDataType; use datatypes::prelude::ScalarVector; use datatypes::type_id::LogicalTypeId; @@ -24,7 +25,6 @@ use store_api::logstore::LogStore; use store_api::storage::{ Chunk, ChunkReader, ReadContext, Region, ScanRequest, Snapshot, WriteContext, WriteRequest, }; -use tempdir::TempDir; use crate::region::{RegionImpl, RegionMetadata}; use crate::test_util::{self, config_util, descriptor_util, write_batch_util}; @@ -179,7 +179,7 @@ async fn new_tester(store_dir: &str) -> ProjectionTester { #[tokio::test] async fn test_projection_ordered() { - let dir = TempDir::new("projection-ordered").unwrap(); + let dir = create_temp_dir("projection-ordered"); let store_dir = dir.path().to_str().unwrap(); let tester = new_tester(store_dir).await; @@ -193,7 +193,7 @@ async fn test_projection_ordered() { #[tokio::test] async fn test_projection_unordered() { - let dir = TempDir::new("projection-unordered").unwrap(); + let dir = create_temp_dir("projection-unordered"); let store_dir = dir.path().to_str().unwrap(); let tester = new_tester(store_dir).await; diff --git a/src/storage/src/sst/parquet.rs b/src/storage/src/sst/parquet.rs index 6aa346670e..fe08d59920 100644 --- a/src/storage/src/sst/parquet.rs +++ b/src/storage/src/sst/parquet.rs @@ -529,6 +529,7 @@ impl BatchReader for ChunkStream { mod tests { use std::sync::Arc; + use common_test_util::temp_dir::create_temp_dir; use datatypes::arrow::array::{Array, ArrayRef, UInt64Array, UInt8Array}; use datatypes::prelude::{ScalarVector, Vector}; use datatypes::types::{TimestampMillisecondType, TimestampType}; @@ -536,7 +537,6 @@ mod tests { use object_store::services::Fs; use object_store::ObjectStoreBuilder; use store_api::storage::OpType; - use tempdir::TempDir; use super::*; use crate::memtable::{ @@ -571,7 +571,7 @@ mod tests { ], // values ); - let dir = TempDir::new("write_parquet").unwrap(); + let dir = create_temp_dir("write_parquet"); let path = dir.path().to_str().unwrap(); let backend = Fs::default().root(path).build().unwrap(); let object_store = ObjectStore::new(backend).finish(); @@ -669,7 +669,7 @@ mod tests { &values_vec, // values ); - let dir = TempDir::new("write_parquet").unwrap(); + let dir = create_temp_dir("write_parquet"); let path = dir.path().to_str().unwrap(); let backend = Fs::default().root(path).build().unwrap(); let object_store = ObjectStore::new(backend).finish(); @@ -746,7 +746,7 @@ mod tests { ], // values ); - let dir = TempDir::new("write_parquet").unwrap(); + let dir = create_temp_dir("write_parquet"); let path = dir.path().to_str().unwrap(); let backend = Fs::default().root(path).build().unwrap(); let object_store = ObjectStore::new(backend).finish(); @@ -863,7 +863,7 @@ mod tests { ], // values ); - let dir = TempDir::new("read-parquet-by-range").unwrap(); + let dir = create_temp_dir("read-parquet-by-range"); let path = dir.path().to_str().unwrap(); let backend = Fs::default().root(path).build().unwrap(); let object_store = ObjectStore::new(backend).finish(); diff --git a/src/storage/src/wal.rs b/src/storage/src/wal.rs index 91920ec270..fa415bd3db 100644 --- a/src/storage/src/wal.rs +++ b/src/storage/src/wal.rs @@ -244,14 +244,17 @@ impl Decoder for WalHeaderDecoder { #[cfg(test)] mod tests { + use common_test_util::temp_dir::create_temp_dir; use log_store::test_util; use super::*; #[tokio::test] pub async fn test_write_wal() { - let (log_store, _tmp) = - test_util::log_store_util::create_tmp_local_file_log_store("wal_test").await; + let log_file_dir = create_temp_dir("wal_test"); + let log_file_dir_path = log_file_dir.path().to_str().unwrap(); + let log_store = + test_util::log_store_util::create_tmp_local_file_log_store(log_file_dir_path).await; let wal = Wal::new(0, Arc::new(log_store)); let res = wal.write(0, b"test1").await.unwrap(); @@ -264,8 +267,10 @@ mod tests { #[tokio::test] pub async fn test_read_wal_only_header() -> Result<()> { common_telemetry::init_default_ut_logging(); - let (log_store, _tmp) = - test_util::log_store_util::create_tmp_local_file_log_store("wal_test").await; + let log_file_dir = create_temp_dir("wal_test"); + let log_file_dir_path = log_file_dir.path().to_str().unwrap(); + let log_store = + test_util::log_store_util::create_tmp_local_file_log_store(log_file_dir_path).await; let wal = Wal::new(0, Arc::new(log_store)); let header = WalHeader::with_last_manifest_version(111); let seq_num = 3; diff --git a/src/table-procedure/Cargo.toml b/src/table-procedure/Cargo.toml index ac813376d8..d37d98ac3b 100644 --- a/src/table-procedure/Cargo.toml +++ b/src/table-procedure/Cargo.toml @@ -17,9 +17,9 @@ snafu.workspace = true table = { path = "../table" } [dev-dependencies] +common-test-util = { path = "../common/test-util" } log-store = { path = "../log-store" } mito = { path = "../mito" } object-store = { path = "../object-store" } storage = { path = "../storage" } tokio.workspace = true -tempdir = "0.3" diff --git a/src/table-procedure/src/test_util.rs b/src/table-procedure/src/test_util.rs index ac047a93f5..c64baab48a 100644 --- a/src/table-procedure/src/test_util.rs +++ b/src/table-procedure/src/test_util.rs @@ -18,6 +18,7 @@ use catalog::local::MemoryCatalogManager; use catalog::CatalogManagerRef; use common_procedure::local::{LocalManager, ManagerConfig}; use common_procedure::ProcedureManagerRef; +use common_test_util::temp_dir::{create_temp_dir, TempDir}; use log_store::NoopLogStore; use mito::config::EngineConfig; use mito::engine::MitoEngine; @@ -26,7 +27,6 @@ use object_store::{ObjectStore, ObjectStoreBuilder}; use storage::compaction::noop::NoopCompactionScheduler; use storage::config::EngineConfig as StorageEngineConfig; use storage::EngineImpl; -use tempdir::TempDir; pub struct TestEnv { pub dir: TempDir, @@ -37,7 +37,7 @@ pub struct TestEnv { impl TestEnv { pub fn new(prefix: &str) -> TestEnv { - let dir = TempDir::new(prefix).unwrap(); + let dir = create_temp_dir(prefix); let store_dir = format!("{}/db", dir.path().to_string_lossy()); let accessor = Fs::default().root(&store_dir).build().unwrap(); let object_store = ObjectStore::new(accessor).finish(); diff --git a/src/table/Cargo.toml b/src/table/Cargo.toml index f4d7918320..41c44cb6e3 100644 --- a/src/table/Cargo.toml +++ b/src/table/Cargo.toml @@ -32,7 +32,7 @@ store-api = { path = "../store-api" } tokio.workspace = true [dev-dependencies] +common-test-util = { path = "../common/test-util" } parquet = { workspace = true, features = ["async"] } -tempdir = "0.3" tokio-util = { version = "0.7", features = ["compat"] } serde_json.workspace = true diff --git a/src/table/src/predicate.rs b/src/table/src/predicate.rs index ec1fb5951b..b6ede3491b 100644 --- a/src/table/src/predicate.rs +++ b/src/table/src/predicate.rs @@ -255,6 +255,7 @@ impl<'a> TimeRangePredicateBuilder<'a> { mod tests { use std::sync::Arc; + use common_test_util::temp_dir::{create_temp_dir, TempDir}; use datafusion::parquet::arrow::ArrowWriter; pub use datafusion::parquet::schema::types::BasicTypeInfo; use datafusion_common::{Column, ScalarValue}; @@ -265,7 +266,6 @@ mod tests { use datatypes::arrow_array::StringArray; use parquet::arrow::ParquetRecordBatchStreamBuilder; use parquet::file::properties::WriterProperties; - use tempdir::TempDir; use super::*; @@ -308,7 +308,7 @@ mod tests { } async fn assert_prune(array_cnt: usize, predicate: Predicate, expect: Vec) { - let dir = TempDir::new("prune_parquet").unwrap(); + let dir = create_temp_dir("prune_parquet"); let (path, schema) = gen_test_parquet_file(&dir, array_cnt).await; let schema = Arc::new(datatypes::schema::Schema::try_from(schema).unwrap()); let builder = ParquetRecordBatchStreamBuilder::new( diff --git a/tests-integration/Cargo.toml b/tests-integration/Cargo.toml index 3e10363ad4..669ceb6c68 100644 --- a/tests-integration/Cargo.toml +++ b/tests-integration/Cargo.toml @@ -16,6 +16,7 @@ common-grpc = { path = "../src/common/grpc" } common-query = { path = "../src/common/query" } common-runtime = { path = "../src/common/runtime" } common-telemetry = { path = "../src/common/telemetry" } +common-test-util = { path = "../src/common/test-util" } datanode = { path = "../src/datanode" } datatypes = { path = "../src/datatypes" } dotenv = "0.15" @@ -30,7 +31,7 @@ servers = { path = "../src/servers" } snafu.workspace = true sql = { path = "../src/sql" } table = { path = "../src/table" } -tempdir = "0.3" +tempfile.workspace = true tokio.workspace = true uuid.workspace = true diff --git a/tests-integration/src/test_util.rs b/tests-integration/src/test_util.rs index 79cdc7173d..ff2b804b00 100644 --- a/tests-integration/src/test_util.rs +++ b/tests-integration/src/test_util.rs @@ -22,6 +22,7 @@ use axum::Router; use catalog::CatalogManagerRef; use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, MIN_USER_TABLE_ID}; use common_runtime::Builder as RuntimeBuilder; +use common_test_util::temp_dir::{create_temp_dir, TempDir}; use datanode::datanode::{ DatanodeOptions, FileConfig, ObjectStoreConfig, OssConfig, S3Config, WalConfig, }; @@ -46,7 +47,6 @@ use servers::Mode; use snafu::ResultExt; use table::engine::{EngineContext, TableEngineRef}; use table::requests::{CreateTableRequest, TableOptions}; -use tempdir::TempDir; static PORTS: OnceCell = OnceCell::new(); @@ -149,7 +149,7 @@ fn get_test_store_config( (config, Some(TempDirGuard::S3(TempFolder::new(&store, "/")))) } StorageType::File => { - let data_tmp_dir = TempDir::new(&format!("gt_data_{name}")).unwrap(); + let data_tmp_dir = create_temp_dir(&format!("gt_data_{name}")); ( ObjectStoreConfig::File(FileConfig { @@ -189,7 +189,7 @@ pub fn create_tmp_dir_and_datanode_opts( store_type: StorageType, name: &str, ) -> (DatanodeOptions, TestGuard) { - let wal_tmp_dir = TempDir::new(&format!("gt_wal_{name}")).unwrap(); + let wal_tmp_dir = create_temp_dir(&format!("gt_wal_{name}")); let (storage, data_tmp_dir) = get_test_store_config(&store_type, name);