rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-05-31 12:20:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: update datafusion family (#6675)

Browse Source 
* chore: update datafusion family

Signed-off-by: luofucong <luofc@foxmail.com>

* fix ci

Signed-off-by: luofucong <luofc@foxmail.com>

* use official otel-arrow-rust

Signed-off-by: luofucong <luofc@foxmail.com>

* rebase

Signed-off-by: luofucong <luofc@foxmail.com>

* use the official orc-rust

Signed-off-by: luofucong <luofc@foxmail.com>

* resolve PR comments

Signed-off-by: luofucong <luofc@foxmail.com>

* remove the empty lines

Signed-off-by: luofucong <luofc@foxmail.com>

* try following PR comments

Signed-off-by: luofucong <luofc@foxmail.com>

---------

Signed-off-by: luofucong <luofc@foxmail.com>
This commit is contained in:
LFC
2025-08-15 20:41:49 +08:00
committed by GitHub
parent dfc29eb3b3
commit f9d2a89a0c
198 changed files with 4932 additions and 4196 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/object-store/Cargo.toml
View File

@@ -21,7 +21,7 @@ humantime-serde.workspace = true
lazy_static.workspace = true
md5 = "0.7"
moka = { workspace = true, features = ["future"] }
opendal = { version = "0.52", features = [
opendal = { git = "https://github.com/apache/opendal", rev = "0ba8574b6d08d209056704d28a9a114beb3c1022", features = [
"layers-tracing",
"layers-prometheus",
"services-azblob",

38
src/object-store/src/factory.rs
View File

@@ -81,16 +81,17 @@ pub async fn new_azblob_object_store(azblob_config: &AzblobConfig) -> Result<Obj
.container(&azblob_config.container)
.endpoint(&azblob_config.endpoint)
.account_name(azblob_config.account_name.expose_secret())
.account_key(azblob_config.account_key.expose_secret())
.http_client(client);
.account_key(azblob_config.account_key.expose_secret());
if let Some(token) = &azblob_config.sas_token {
builder = builder.sas_token(token);
};
Ok(ObjectStore::new(builder)
let operator = ObjectStore::new(builder)
.context(error::InitBackendSnafu)?
.finish())
.finish();
operator.update_http_client(|_| client);
Ok(operator)
}
pub async fn new_gcs_object_store(gcs_config: &GcsConfig) -> Result<ObjectStore> {
@@ -100,7 +101,7 @@ pub async fn new_gcs_object_store(gcs_config: &GcsConfig) -> Result<ObjectStore>
gcs_config.bucket, &root
);
let client = build_http_client(&gcs_config.http_client);
let client = build_http_client(&gcs_config.http_client)?;
let builder = Gcs::default()
.root(&root)
@@ -108,12 +109,13 @@ pub async fn new_gcs_object_store(gcs_config: &GcsConfig) -> Result<ObjectStore>
.scope(&gcs_config.scope)
.credential_path(gcs_config.credential_path.expose_secret())
.credential(gcs_config.credential.expose_secret())
.endpoint(&gcs_config.endpoint)
.http_client(client?);
.endpoint(&gcs_config.endpoint);
Ok(ObjectStore::new(builder)
let operator = ObjectStore::new(builder)
.context(error::InitBackendSnafu)?
.finish())
.finish();
operator.update_http_client(|_| client);
Ok(operator)
}
pub async fn new_oss_object_store(oss_config: &OssConfig) -> Result<ObjectStore> {
@@ -130,12 +132,13 @@ pub async fn new_oss_object_store(oss_config: &OssConfig) -> Result<ObjectStore>
.bucket(&oss_config.bucket)
.endpoint(&oss_config.endpoint)
.access_key_id(oss_config.access_key_id.expose_secret())
.access_key_secret(oss_config.access_key_secret.expose_secret())
.http_client(client);
.access_key_secret(oss_config.access_key_secret.expose_secret());
Ok(ObjectStore::new(builder)
let operator = ObjectStore::new(builder)
.context(error::InitBackendSnafu)?
.finish())
.finish();
operator.update_http_client(|_| client);
Ok(operator)
}
pub async fn new_s3_object_store(s3_config: &S3Config) -> Result<ObjectStore> {
@@ -152,8 +155,7 @@ pub async fn new_s3_object_store(s3_config: &S3Config) -> Result<ObjectStore> {
.root(&root)
.bucket(&s3_config.bucket)
.access_key_id(s3_config.access_key_id.expose_secret())
.secret_access_key(s3_config.secret_access_key.expose_secret())
.http_client(client);
.secret_access_key(s3_config.secret_access_key.expose_secret());
if s3_config.endpoint.is_some() {
builder = builder.endpoint(s3_config.endpoint.as_ref().unwrap());
@@ -165,7 +167,9 @@ pub async fn new_s3_object_store(s3_config: &S3Config) -> Result<ObjectStore> {
builder = builder.enable_virtual_host_style();
}
Ok(ObjectStore::new(builder)
let operator = ObjectStore::new(builder)
.context(error::InitBackendSnafu)?
.finish())
.finish();
operator.update_http_client(|_| client);
Ok(operator)
}

6
src/object-store/src/layers.rs
View File

@@ -40,11 +40,7 @@ mod prometheus {
// remove path label first, waiting for later fix
// TODO(shuiyisong): add dynamic path label trim for opendal
let layer = PrometheusLayer::builder()
.path_label(0)
.register_default()
.unwrap();
let layer = PrometheusLayer::builder().register_default().unwrap();
Mutex::new(layer)
})
.lock()

27
src/object-store/src/util.rs
View File

@@ -16,7 +16,7 @@ use std::fmt::Display;
use std::path;
use std::time::Duration;
use common_telemetry::{debug, error, info, trace, warn};
use common_telemetry::{debug, error, info, warn};
use opendal::layers::{LoggingInterceptor, LoggingLayer, RetryInterceptor, TracingLayer};
use opendal::raw::{AccessorInfo, HttpClient, Operation};
use opendal::{Error, ErrorKind};
@@ -183,24 +183,13 @@ impl LoggingInterceptor for DefaultLoggingInterceptor {
};
}
// Print debug message if operation is oneshot, otherwise in trace.
if operation.is_oneshot() {
debug!(
target: LOGGING_TARGET,
"service={} name={} {}: {operation} {message}",
info.scheme(),
info.name(),
LoggingContext(context),
);
} else {
trace!(
target: LOGGING_TARGET,
"service={} name={} {}: {operation} {message}",
info.scheme(),
info.name(),
LoggingContext(context),
);
};
debug!(
target: LOGGING_TARGET,
"service={} name={} {}: {operation} {message}",
info.scheme(),
info.name(),
LoggingContext(context),
);
}
}