rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-05-21 23:40:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

build(deps): upgrade opendal to 0.46 (#4037)

Browse Source 
* build(deps): upgrade opendal to 0.46

Signed-off-by: tison <wander4096@gmail.com>

* migrate writes

Signed-off-by: tison <wander4096@gmail.com>

* migrate reads

Signed-off-by: tison <wander4096@gmail.com>

* fixup object safety

Signed-off-by: tison <wander4096@gmail.com>

* fixup names

Signed-off-by: tison <wander4096@gmail.com>

* fixup compilation

Signed-off-by: tison <wander4096@gmail.com>

* fixup compilation

Signed-off-by: tison <wander4096@gmail.com>

* a few Buffer to Vec

Signed-off-by: tison <wander4096@gmail.com>

* Make greptime buildable with opendal 0.46 (#5)

Signed-off-by: Xuanwo <github@xuanwo.io>

* fixup toml check

Signed-off-by: tison <wander4096@gmail.com>

* test_orc_opener

Signed-off-by: tison <wander4096@gmail.com>

* Fix lru cache (#6)

Signed-off-by: Xuanwo <github@xuanwo.io>

* clippy

Signed-off-by: tison <wander4096@gmail.com>

* improve comments

Signed-off-by: tison <wander4096@gmail.com>

* address comments

Signed-off-by: tison <wander4096@gmail.com>

* reduce buf copy

Signed-off-by: tison <wander4096@gmail.com>

* upgrade to reqwest 0.12

Signed-off-by: tison <wander4096@gmail.com>

---------

Signed-off-by: tison <wander4096@gmail.com>
Signed-off-by: Xuanwo <github@xuanwo.io>
Co-authored-by: Xuanwo <github@xuanwo.io>
This commit is contained in:
tison
2024-05-27 17:12:23 +08:00
committed by GitHub
parent 20ce7d428d
commit f9db5ff0d6
31 changed files with 631 additions and 443 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
src/object-store/tests/object_store_test.rs
View File

@@ -22,7 +22,6 @@ use object_store::layers::LruCacheLayer;
use object_store::services::{Fs, S3};
use object_store::test_util::TempFolder;
use object_store::{ObjectStore, ObjectStoreBuilder};
use opendal::raw::Accessor;
use opendal::services::{Azblob, Gcs, Oss};
use opendal::{EntryMode, Operator, OperatorBuilder};
@@ -36,11 +35,11 @@ async fn test_object_crud(store: &ObjectStore) -> Result<()> {
// Read data from object;
let bs = store.read(file_name).await?;
assert_eq!("Hello, World!", String::from_utf8(bs)?);
assert_eq!("Hello, World!", String::from_utf8(bs.to_vec())?);
// Read range from object;
let bs = store.read_with(file_name).range(1..=11).await?;
assert_eq!("ello, World", String::from_utf8(bs)?);
assert_eq!("ello, World", String::from_utf8(bs.to_vec())?);
// Get object's Metadata
let meta = store.stat(file_name).await?;
@@ -77,7 +76,7 @@ async fn test_object_list(store: &ObjectStore) -> Result<()> {
assert_eq!(p2, entries.first().unwrap().path());
let content = store.read(p2).await?;
assert_eq!("Hello, object2!", String::from_utf8(content)?);
assert_eq!("Hello, object2!", String::from_utf8(content.to_vec())?);
store.delete(p2).await?;
let entries = store.list("/").await?;
@@ -236,11 +235,9 @@ async fn test_file_backend_with_lru_cache() -> Result<()> {
let _ = builder
.root(&cache_dir.path().to_string_lossy())
.atomic_write_dir(&cache_dir.path().to_string_lossy());
let file_cache = Arc::new(builder.build().unwrap());
let file_cache = Operator::new(builder).unwrap().finish();
LruCacheLayer::new(Arc::new(file_cache.clone()), 32)
.await
.unwrap()
LruCacheLayer::new(file_cache, 32).await.unwrap()
};
let store = store.layer(cache_layer.clone());
@@ -253,10 +250,7 @@ async fn test_file_backend_with_lru_cache() -> Result<()> {
Ok(())
}
async fn assert_lru_cache<C: Accessor + Clone>(
cache_layer: &LruCacheLayer<C>,
file_names: &[&str],
) {
async fn assert_lru_cache(cache_layer: &LruCacheLayer, file_names: &[&str]) {
for file_name in file_names {
assert!(cache_layer.contains_file(file_name).await);
}
@@ -278,7 +272,7 @@ async fn assert_cache_files(
let bs = store.read(o.path()).await.unwrap();
assert_eq!(
file_contents[position],
String::from_utf8(bs.clone())?,
String::from_utf8(bs.to_vec())?,
"file content not match: {}",
o.name()
);
@@ -312,9 +306,7 @@ async fn test_object_store_cache_policy() -> Result<()> {
let cache_store = OperatorBuilder::new(file_cache.clone()).finish();
// create operator for cache dir to verify cache file
let cache_layer = LruCacheLayer::new(Arc::new(file_cache.clone()), 38)
.await
.unwrap();
let cache_layer = LruCacheLayer::new(cache_store.clone(), 38).await.unwrap();
let store = store.layer(cache_layer.clone());
// create several object handler.
@@ -386,7 +378,7 @@ async fn test_object_store_cache_policy() -> Result<()> {
// instead of returning `NotFound` during the reader creation.
// The entry count is 4, because we have the p2 `NotFound` cache.
assert!(store.read_with(p2).range(0..4).await.is_err());
assert_eq!(cache_layer.read_cache_stat().await, (4, 35));
assert_eq!(cache_layer.read_cache_stat().await, (3, 35));
assert_cache_files(
&cache_store,
@@ -414,7 +406,7 @@ async fn test_object_store_cache_policy() -> Result<()> {
assert!(store.read(p2).await.is_err());
// Read p1 with range `1..` , the existing p1 with range `0..` must be evicted.
let _ = store.read_with(p1).range(1..15).await.unwrap();
assert_eq!(cache_layer.read_cache_stat().await, (4, 34));
assert_eq!(cache_layer.read_cache_stat().await, (3, 34));
assert_cache_files(
&cache_store,
&[
@@ -442,7 +434,7 @@ async fn test_object_store_cache_policy() -> Result<()> {
drop(cache_layer);
// Test recover
let cache_layer = LruCacheLayer::new(Arc::new(file_cache), 38).await.unwrap();
let cache_layer = LruCacheLayer::new(cache_store, 38).await.unwrap();
// The p2 `NotFound` cache will not be recovered
assert_eq!(cache_layer.read_cache_stat().await, (3, 34));