rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-05-25 17:30:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: upgrade opendal (#1245)

Browse Source 
* chore: upgrade opendal

* chore: finish upgrading opendal

* fix: clippy complaints

* fix some tests

* fix: all unit tests

* chore: rebase develop

* fix: sqlness tests

* optimize imports

* chore: rebase develop

* doc: add todo
This commit is contained in:
Lei, HUANG
2023-03-28 09:47:33 +08:00
committed by GitHub
parent e63b28bff1
commit 5edd2a3dbe
31 changed files with 390 additions and 431 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
src/datanode/src/instance.rs
View File

@@ -330,18 +330,20 @@ pub(crate) async fn new_oss_object_store(store_config: &ObjectStoreConfig) -> Re
);
let mut builder = OSSBuilder::default();
let builder = builder
builder
.root(&root)
.bucket(&oss_config.bucket)
.endpoint(&oss_config.endpoint)
.access_key_id(&oss_config.access_key_id)
.access_key_secret(&oss_config.access_key_secret);
let accessor = builder.build().with_context(|_| error::InitBackendSnafu {
config: store_config.clone(),
})?;
let object_store = ObjectStore::new(builder)
.with_context(|_| error::InitBackendSnafu {
config: store_config.clone(),
})?
.finish();
create_object_store_with_cache(ObjectStore::new(accessor).finish(), store_config)
create_object_store_with_cache(object_store, store_config)
}
fn create_object_store_with_cache(
@@ -394,24 +396,27 @@ pub(crate) async fn new_s3_object_store(store_config: &ObjectStoreConfig) -> Res
);
let mut builder = S3Builder::default();
let mut builder = builder
builder
.root(&root)
.bucket(&s3_config.bucket)
.access_key_id(&s3_config.access_key_id)
.secret_access_key(&s3_config.secret_access_key);
if s3_config.endpoint.is_some() {
builder = builder.endpoint(s3_config.endpoint.as_ref().unwrap());
builder.endpoint(s3_config.endpoint.as_ref().unwrap());
}
if s3_config.region.is_some() {
builder = builder.region(s3_config.region.as_ref().unwrap());
builder.region(s3_config.region.as_ref().unwrap());
}
let accessor = builder.build().with_context(|_| error::InitBackendSnafu {
config: store_config.clone(),
})?;
create_object_store_with_cache(ObjectStore::new(accessor).finish(), store_config)
create_object_store_with_cache(
ObjectStore::new(builder)
.with_context(|_| error::InitBackendSnafu {
config: store_config.clone(),
})?
.finish(),
store_config,
)
}
pub(crate) async fn new_fs_object_store(store_config: &ObjectStoreConfig) -> Result<ObjectStore> {
@@ -426,15 +431,14 @@ pub(crate) async fn new_fs_object_store(store_config: &ObjectStoreConfig) -> Res
let atomic_write_dir = format!("{data_dir}/.tmp/");
let accessor = FsBuilder::default()
.root(&data_dir)
.atomic_write_dir(&atomic_write_dir)
.build()
let mut builder = FsBuilder::default();
builder.root(&data_dir).atomic_write_dir(&atomic_write_dir);
Ok(ObjectStore::new(builder)
.context(error::InitBackendSnafu {
config: store_config.clone(),
})?;
Ok(ObjectStore::new(accessor).finish())
})?
.finish())
}
/// Create metasrv client instance and spawn heartbeat loop.

15
src/datanode/src/sql/copy_table_from.rs
View File

@@ -48,7 +48,6 @@ impl SqlHandler {
build_backend(&req.location, req.connection).context(error::BuildBackendSnafu)?;
let (dir, filename) = find_dir_and_filename(&path);
let regex = req
.pattern
.as_ref()
@@ -62,16 +61,18 @@ impl SqlHandler {
Source::Dir
};
let lister = Lister::new(object_store, source, dir, regex);
let lister = Lister::new(object_store.clone(), source, dir, regex);
let objects = lister.list().await.context(error::ListObjectsSnafu)?;
let entries = lister.list().await.context(error::ListObjectsSnafu)?;
let mut buf: Vec<RecordBatch> = Vec::new();
for obj in objects.iter() {
let reader = obj.reader().await.context(error::ReadObjectSnafu {
path: &obj.path().to_string(),
})?;
for entry in entries.iter() {
let path = entry.path();
let reader = object_store
.reader(path)
.await
.context(error::ReadObjectSnafu { path })?;
let buf_reader = BufReader::new(reader.compat());

8
src/datanode/src/sql/copy_table_to.rs
View File

@@ -136,10 +136,10 @@ impl ParquetWriter {
// "file_name_1_1000000" (row num: 1 ~ 1000000),
// "file_name_1000001_xxx" (row num: 1000001 ~ xxx)
let file_name = format!("{}_{}_{}", self.file_name, start_row_num, total_rows);
let object = self.object_store.object(&file_name);
object.write(buf).await.context(error::WriteObjectSnafu {
path: object.path(),
})?;
self.object_store
.write(&file_name, buf)
.await
.context(error::WriteObjectSnafu { path: file_name })?;
if end_loop {
return Ok(total_rows);