rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2025-12-27 16:32:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(cli): refactor object storage config (#7009)

Browse Source 
* refactor: refactor object storage config

Signed-off-by: WenyXu <wenymedia@gmail.com>

* chore: public common config

Signed-off-by: WenyXu <wenymedia@gmail.com>

* chore: apply suggestions

Signed-off-by: WenyXu <wenymedia@gmail.com>

---------

Signed-off-by: WenyXu <wenymedia@gmail.com>
This commit is contained in:
Weny Xu
2025-09-24 14:50:47 +08:00
committed by GitHub
parent b5a8725582
commit 0c038f755f
23 changed files with 546 additions and 462 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
tests-integration/src/test_util.rs
View File

@@ -20,7 +20,6 @@ use std::sync::Arc;
use auth::UserProviderRef;
use axum::Router;
use catalog::kvbackend::KvBackendCatalogManager;
use common_base::secrets::ExposeSecret;
use common_config::Configurable;
use common_meta::key::catalog_name::CatalogNameKey;
use common_meta::key::schema_name::SchemaNameKey;
@@ -32,12 +31,12 @@ use common_wal::config::DatanodeWalConfig;
use datanode::config::{DatanodeOptions, StorageConfig};
use frontend::instance::Instance;
use frontend::service_config::{MysqlOptions, PostgresOptions};
use object_store::ObjectStore;
use object_store::config::{
AzblobConfig, FileConfig, GcsConfig, ObjectStoreConfig, OssConfig, S3Config,
};
use object_store::services::{Azblob, Gcs, Oss, S3};
use object_store::test_util::TempFolder;
use object_store::{AzblobConnection, GcsConnection, ObjectStore, OssConnection, S3Connection};
use servers::grpc::builder::GrpcServerBuilder;
use servers::grpc::greptime_handler::GreptimeRequestHandler;
use servers::grpc::{FlightCompression, GrpcOptions, GrpcServer, GrpcServerConfig};
@@ -139,11 +138,14 @@ impl StorageType {
fn s3_test_config() -> S3Config {
S3Config {
root: uuid::Uuid::new_v4().to_string(),
access_key_id: env::var("GT_S3_ACCESS_KEY_ID").unwrap().into(),
secret_access_key: env::var("GT_S3_ACCESS_KEY").unwrap().into(),
bucket: env::var("GT_S3_BUCKET").unwrap(),
region: Some(env::var("GT_S3_REGION").unwrap()),
connection: S3Connection {
root: uuid::Uuid::new_v4().to_string(),
access_key_id: env::var("GT_S3_ACCESS_KEY_ID").unwrap().into(),
secret_access_key: env::var("GT_S3_ACCESS_KEY").unwrap().into(),
bucket: env::var("GT_S3_BUCKET").unwrap(),
region: Some(env::var("GT_S3_REGION").unwrap()),
..Default::default()
},
..Default::default()
}
}
@@ -154,75 +156,55 @@ pub fn get_test_store_config(store_type: &StorageType) -> (ObjectStoreConfig, Te
match store_type {
StorageType::Gcs => {
let gcs_config = GcsConfig {
root: uuid::Uuid::new_v4().to_string(),
bucket: env::var("GT_GCS_BUCKET").unwrap(),
scope: env::var("GT_GCS_SCOPE").unwrap(),
credential_path: env::var("GT_GCS_CREDENTIAL_PATH").unwrap().into(),
credential: env::var("GT_GCS_CREDENTIAL").unwrap().into(),
endpoint: env::var("GT_GCS_ENDPOINT").unwrap(),
connection: GcsConnection {
root: uuid::Uuid::new_v4().to_string(),
bucket: env::var("GT_GCS_BUCKET").unwrap(),
scope: env::var("GT_GCS_SCOPE").unwrap(),
credential_path: env::var("GT_GCS_CREDENTIAL_PATH").unwrap().into(),
credential: env::var("GT_GCS_CREDENTIAL").unwrap().into(),
endpoint: env::var("GT_GCS_ENDPOINT").unwrap(),
},
..Default::default()
};
let builder = Gcs::default()
.root(&gcs_config.root)
.bucket(&gcs_config.bucket)
.scope(&gcs_config.scope)
.credential_path(gcs_config.credential_path.expose_secret())
.credential(gcs_config.credential.expose_secret())
.endpoint(&gcs_config.endpoint);
let builder = Gcs::from(&gcs_config.connection);
let config = ObjectStoreConfig::Gcs(gcs_config);
let store = ObjectStore::new(builder).unwrap().finish();
(config, TempDirGuard::Gcs(TempFolder::new(&store, "/")))
}
StorageType::Azblob => {
let azblob_config = AzblobConfig {
root: uuid::Uuid::new_v4().to_string(),
container: env::var("GT_AZBLOB_CONTAINER").unwrap(),
account_name: env::var("GT_AZBLOB_ACCOUNT_NAME").unwrap().into(),
account_key: env::var("GT_AZBLOB_ACCOUNT_KEY").unwrap().into(),
endpoint: env::var("GT_AZBLOB_ENDPOINT").unwrap(),
connection: AzblobConnection {
root: uuid::Uuid::new_v4().to_string(),
container: env::var("GT_AZBLOB_CONTAINER").unwrap(),
account_name: env::var("GT_AZBLOB_ACCOUNT_NAME").unwrap().into(),
account_key: env::var("GT_AZBLOB_ACCOUNT_KEY").unwrap().into(),
endpoint: env::var("GT_AZBLOB_ENDPOINT").unwrap(),
..Default::default()
},
..Default::default()
};
let mut builder = Azblob::default()
.root(&azblob_config.root)
.endpoint(&azblob_config.endpoint)
.account_name(azblob_config.account_name.expose_secret())
.account_key(azblob_config.account_key.expose_secret())
.container(&azblob_config.container);
if let Ok(sas_token) = env::var("GT_AZBLOB_SAS_TOKEN") {
builder = builder.sas_token(&sas_token);
};
let builder = Azblob::from(&azblob_config.connection);
let config = ObjectStoreConfig::Azblob(azblob_config);
let store = ObjectStore::new(builder).unwrap().finish();
(config, TempDirGuard::Azblob(TempFolder::new(&store, "/")))
}
StorageType::Oss => {
let oss_config = OssConfig {
root: uuid::Uuid::new_v4().to_string(),
access_key_id: env::var("GT_OSS_ACCESS_KEY_ID").unwrap().into(),
access_key_secret: env::var("GT_OSS_ACCESS_KEY").unwrap().into(),
bucket: env::var("GT_OSS_BUCKET").unwrap(),
endpoint: env::var("GT_OSS_ENDPOINT").unwrap(),
connection: OssConnection {
root: uuid::Uuid::new_v4().to_string(),
access_key_id: env::var("GT_OSS_ACCESS_KEY_ID").unwrap().into(),
access_key_secret: env::var("GT_OSS_ACCESS_KEY").unwrap().into(),
bucket: env::var("GT_OSS_BUCKET").unwrap(),
endpoint: env::var("GT_OSS_ENDPOINT").unwrap(),
},
..Default::default()
};
let builder = Oss::default()
.root(&oss_config.root)
.endpoint(&oss_config.endpoint)
.access_key_id(oss_config.access_key_id.expose_secret())
.access_key_secret(oss_config.access_key_secret.expose_secret())
.bucket(&oss_config.bucket);
let builder = Oss::from(&oss_config.connection);
let config = ObjectStoreConfig::Oss(oss_config);
let store = ObjectStore::new(builder).unwrap().finish();
(config, TempDirGuard::Oss(TempFolder::new(&store, "/")))
}
StorageType::S3 | StorageType::S3WithCache => {
@@ -235,23 +217,9 @@ pub fn get_test_store_config(store_type: &StorageType) -> (ObjectStoreConfig, Te
s3_config.cache.cache_path = Some("".to_string());
}
let mut builder = S3::default()
.root(&s3_config.root)
.access_key_id(s3_config.access_key_id.expose_secret())
.secret_access_key(s3_config.secret_access_key.expose_secret())
.bucket(&s3_config.bucket);
if s3_config.endpoint.is_some() {
builder = builder.endpoint(s3_config.endpoint.as_ref().unwrap());
};
if s3_config.region.is_some() {
builder = builder.region(s3_config.region.as_ref().unwrap());
};
let builder = S3::from(&s3_config.connection);
let config = ObjectStoreConfig::S3(s3_config);
let store = ObjectStore::new(builder).unwrap().finish();
(config, TempDirGuard::S3(TempFolder::new(&store, "/")))
}
StorageType::File => (ObjectStoreConfig::File(FileConfig {}), TempDirGuard::None),