mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-05-23 16:30:39 +00:00
feat: impl object-store based on opendal (#27)
* feat: impl object-store based on opendal * feat: adds s3 test * feat: export Layer * feat: pass s3 access key/secret by env vars * fix: logging when s3 test is running
This commit is contained in:
dennis zhuang
GitHub
3
.github/workflows/coverage.yml
vendored
3
.github/workflows/coverage.yml
vendored
@@ -29,6 +29,9 @@ jobs:
|
||||
env:
|
||||
CARGO_INCREMENTAL: 0
|
||||
RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=abort -Zpanic_abort_tests"
|
||||
GT_S3_BUCKET: ${{ secrets.S3_BUCKET }}
|
||||
GT_S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
|
||||
GT_S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
|
||||
- name: Gather coverage data
|
||||
id: coverage
|
||||
uses: actions-rs/grcov@v0.1
|
||||
|
||||
4
.github/workflows/develop.yml
vendored
4
.github/workflows/develop.yml
vendored
@@ -35,6 +35,10 @@ jobs:
|
||||
with:
|
||||
command: test
|
||||
args: --workspace
|
||||
env:
|
||||
GT_S3_BUCKET: ${{ secrets.S3_BUCKET }}
|
||||
GT_S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
|
||||
GT_S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
|
||||
|
||||
fmt:
|
||||
name: Rustfmt
|
||||
|
||||
436
Cargo.lock
generated
436
Cargo.lock
generated
@@ -127,6 +127,19 @@ dependencies = [
|
||||
"strength_reduce",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-compat"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b48b4ff0c2026db683dea961cd8ea874737f56cffca86fa84415eaddc51c00d"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-io",
|
||||
"once_cell",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-compression"
|
||||
version = "0.3.12"
|
||||
@@ -632,6 +645,20 @@ dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4ae5588f6b3c3cb05239e90bd110f257254aecd01e4635400391aeae07497845"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crossbeam-channel",
|
||||
"crossbeam-deque",
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-queue",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-channel"
|
||||
version = "0.5.4"
|
||||
@@ -642,6 +669,17 @@ dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-deque"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.9.8"
|
||||
@@ -656,6 +694,16 @@ dependencies = [
|
||||
"scopeguard",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-queue"
|
||||
version = "0.3.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1f25d8400f4a7a5778f0e4e52384a48cbd9b5c495d110786187fc750075277a2"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.8.8"
|
||||
@@ -704,6 +752,16 @@ dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ctor"
|
||||
version = "0.1.22"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f877be4f7c9f246b183111634f75baa039715e3f46ce860677d3b19a69fb229c"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "datafusion"
|
||||
version = "7.0.0"
|
||||
@@ -727,7 +785,7 @@ dependencies = [
|
||||
"parquet2",
|
||||
"paste",
|
||||
"pin-project-lite",
|
||||
"rand",
|
||||
"rand 0.8.5",
|
||||
"smallvec",
|
||||
"sqlparser",
|
||||
"tempfile",
|
||||
@@ -774,7 +832,7 @@ dependencies = [
|
||||
"md-5",
|
||||
"ordered-float 2.10.0",
|
||||
"paste",
|
||||
"rand",
|
||||
"rand 0.8.5",
|
||||
"regex",
|
||||
"sha2",
|
||||
"unicode-segmentation",
|
||||
@@ -827,6 +885,32 @@ dependencies = [
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dirs"
|
||||
version = "4.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059"
|
||||
dependencies = [
|
||||
"dirs-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dirs-sys"
|
||||
version = "0.3.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"redox_users",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dlv-list"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257"
|
||||
|
||||
[[package]]
|
||||
name = "doc-comment"
|
||||
version = "0.3.3"
|
||||
@@ -912,6 +996,12 @@ dependencies = [
|
||||
"percent-encoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fuchsia-cprng"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
|
||||
|
||||
[[package]]
|
||||
name = "futures"
|
||||
version = "0.3.21"
|
||||
@@ -1119,6 +1209,21 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hex"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
|
||||
|
||||
[[package]]
|
||||
name = "hmac"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
|
||||
dependencies = [
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "http"
|
||||
version = "0.2.7"
|
||||
@@ -1308,6 +1413,20 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jsonwebtoken"
|
||||
version = "8.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cc9051c17f81bae79440afa041b3a278e1de71bfb96d32454b477fd4703ccb6f"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"pem",
|
||||
"ring",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"simple_asn1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lazy_static"
|
||||
version = "1.4.0"
|
||||
@@ -1470,6 +1589,12 @@ dependencies = [
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "md5"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.5.0"
|
||||
@@ -1559,6 +1684,34 @@ version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
|
||||
|
||||
[[package]]
|
||||
name = "minitrace"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a07fdf302cc0591c97eb45939550ddaddd9962e400c20b319aa16c244cb1f16"
|
||||
dependencies = [
|
||||
"crossbeam",
|
||||
"futures",
|
||||
"minitrace-macro",
|
||||
"minstant",
|
||||
"once_cell",
|
||||
"parking_lot 0.11.2",
|
||||
"pin-project",
|
||||
"retain_mut",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "minitrace-macro"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4132dfe6097f4a90c0bbb34be0687c38d14303dd2e74f8442ae80e9bc5a34c47"
|
||||
dependencies = [
|
||||
"proc-macro-error",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.5.1"
|
||||
@@ -1568,6 +1721,17 @@ dependencies = [
|
||||
"adler",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "minstant"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2cb320648b7883b43ce5dfbc5c6f4a84038194c3f67b4fcb7d05c994e6006557"
|
||||
dependencies = [
|
||||
"ctor",
|
||||
"libc",
|
||||
"wasi 0.7.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mio"
|
||||
version = "0.8.3"
|
||||
@@ -1628,6 +1792,17 @@ dependencies = [
|
||||
"minimal-lexical",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-bigint"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"num-integer",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-integer"
|
||||
version = "0.1.45"
|
||||
@@ -1678,6 +1853,14 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "object-store"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"common-telemetry",
|
||||
"futures",
|
||||
"opendal",
|
||||
"tempdir",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
@@ -1685,6 +1868,38 @@ version = "1.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9"
|
||||
|
||||
[[package]]
|
||||
name = "opendal"
|
||||
version = "0.6.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3649ace5a99d388ac9d02459135ad0425941e8cf6c33f418c4ded80483563ce3"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-compat",
|
||||
"async-trait",
|
||||
"base64",
|
||||
"bytes",
|
||||
"futures",
|
||||
"http",
|
||||
"hyper",
|
||||
"hyper-tls",
|
||||
"log",
|
||||
"md5",
|
||||
"metrics",
|
||||
"minitrace",
|
||||
"once_cell",
|
||||
"parking_lot 0.12.0",
|
||||
"pin-project",
|
||||
"quick-xml",
|
||||
"reqsign",
|
||||
"reqwest",
|
||||
"roxmltree",
|
||||
"serde",
|
||||
"thiserror",
|
||||
"time 0.3.9",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "openssl"
|
||||
version = "0.10.40"
|
||||
@@ -1745,7 +1960,7 @@ dependencies = [
|
||||
"lazy_static",
|
||||
"percent-encoding",
|
||||
"pin-project",
|
||||
"rand",
|
||||
"rand 0.8.5",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
@@ -1793,6 +2008,16 @@ dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ordered-multimap"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a"
|
||||
dependencies = [
|
||||
"dlv-list",
|
||||
"hashbrown 0.12.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "os_str_bytes"
|
||||
version = "6.0.0"
|
||||
@@ -1887,6 +2112,15 @@ version = "1.0.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0c520e05135d6e763148b6426a837e239041653ba7becd2e538c076c738025fc"
|
||||
|
||||
[[package]]
|
||||
name = "pem"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e9a3b09a20e374558580a4914d3b7d89bd61b954a5a5e1dcbea98753addb1947"
|
||||
dependencies = [
|
||||
"base64",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "percent-encoding"
|
||||
version = "2.1.0"
|
||||
@@ -2059,6 +2293,25 @@ dependencies = [
|
||||
"tokio-stream",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quick-xml"
|
||||
version = "0.22.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8533f14c8382aaad0d592c812ac3b826162128b65662331e1127b45c3d18536b"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quickcheck"
|
||||
version = "1.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6"
|
||||
dependencies = [
|
||||
"rand 0.8.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.18"
|
||||
@@ -2068,6 +2321,19 @@ dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.4.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293"
|
||||
dependencies = [
|
||||
"fuchsia-cprng",
|
||||
"libc",
|
||||
"rand_core 0.3.1",
|
||||
"rdrand",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.8.5"
|
||||
@@ -2076,7 +2342,7 @@ checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"rand_chacha",
|
||||
"rand_core",
|
||||
"rand_core 0.6.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2086,9 +2352,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
|
||||
dependencies = [
|
||||
"ppv-lite86",
|
||||
"rand_core",
|
||||
"rand_core 0.6.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b"
|
||||
dependencies = [
|
||||
"rand_core 0.4.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc"
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.6.3"
|
||||
@@ -2107,6 +2388,15 @@ dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rdrand"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2"
|
||||
dependencies = [
|
||||
"rand_core 0.3.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.2.13"
|
||||
@@ -2116,6 +2406,17 @@ dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_users"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
"redox_syscall",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.5.5"
|
||||
@@ -2151,6 +2452,35 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "reqsign"
|
||||
version = "0.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8931679eac96ffc8eee4e5507c4b91fbc0799f29a6535707ee3ef89c0d0de426"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
"base64",
|
||||
"bytes",
|
||||
"dirs",
|
||||
"form_urlencoded",
|
||||
"hex",
|
||||
"hmac",
|
||||
"http",
|
||||
"jsonwebtoken",
|
||||
"log",
|
||||
"once_cell",
|
||||
"percent-encoding",
|
||||
"reqwest",
|
||||
"roxmltree",
|
||||
"rust-ini",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"time 0.3.9",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "reqwest"
|
||||
version = "0.11.10"
|
||||
@@ -2189,6 +2519,46 @@ dependencies = [
|
||||
"winreg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "retain_mut"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8c31b5c4033f8fdde8700e4657be2c497e7288f01515be52168c631e2e4d4086"
|
||||
|
||||
[[package]]
|
||||
name = "ring"
|
||||
version = "0.16.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
"once_cell",
|
||||
"spin",
|
||||
"untrusted",
|
||||
"web-sys",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "roxmltree"
|
||||
version = "0.14.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "921904a62e410e37e215c40381b7117f830d9d89ba60ab5236170541dd25646b"
|
||||
dependencies = [
|
||||
"xmlparser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rust-ini"
|
||||
version = "0.18.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"ordered-multimap",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-demangle"
|
||||
version = "0.1.21"
|
||||
@@ -2325,6 +2695,18 @@ version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a"
|
||||
|
||||
[[package]]
|
||||
name = "simple_asn1"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4a762b1c38b9b990c694b9c2f8abe3372ce6a9ceaae6bca39cfc46e054f45745"
|
||||
dependencies = [
|
||||
"num-bigint",
|
||||
"num-traits",
|
||||
"thiserror",
|
||||
"time 0.3.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sketches-ddsketch"
|
||||
version = "0.1.2"
|
||||
@@ -2382,6 +2764,12 @@ dependencies = [
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "spin"
|
||||
version = "0.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
|
||||
|
||||
[[package]]
|
||||
name = "sql"
|
||||
version = "0.1.0"
|
||||
@@ -2493,6 +2881,16 @@ dependencies = [
|
||||
"snafu",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tempdir"
|
||||
version = "0.3.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8"
|
||||
dependencies = [
|
||||
"rand 0.4.6",
|
||||
"remove_dir_all",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tempfile"
|
||||
version = "3.3.0"
|
||||
@@ -2603,8 +3001,16 @@ dependencies = [
|
||||
"itoa 1.0.1",
|
||||
"libc",
|
||||
"num_threads",
|
||||
"quickcheck",
|
||||
"time-macros",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "time-macros"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792"
|
||||
|
||||
[[package]]
|
||||
name = "tinyvec"
|
||||
version = "1.6.0"
|
||||
@@ -2755,7 +3161,7 @@ dependencies = [
|
||||
"indexmap",
|
||||
"pin-project",
|
||||
"pin-project-lite",
|
||||
"rand",
|
||||
"rand 0.8.5",
|
||||
"slab",
|
||||
"tokio",
|
||||
"tokio-util 0.7.1",
|
||||
@@ -2976,6 +3382,12 @@ version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04"
|
||||
|
||||
[[package]]
|
||||
name = "untrusted"
|
||||
version = "0.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
|
||||
|
||||
[[package]]
|
||||
name = "url"
|
||||
version = "2.2.2"
|
||||
@@ -3025,6 +3437,12 @@ dependencies = [
|
||||
"try-lock",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b89c3ce4ce14bdc6fb6beaf9ec7928ca331de5df7e5ea278375642a2f478570d"
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.10.2+wasi-snapshot-preview1"
|
||||
@@ -3196,6 +3614,12 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "xmlparser"
|
||||
version = "0.13.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "114ba2b24d2167ef6d67d7d04c8cc86522b87f490025f39f0303b7db5bf5e3d8"
|
||||
|
||||
[[package]]
|
||||
name = "zstd"
|
||||
version = "0.10.0+zstd.1.5.2"
|
||||
|
||||
@@ -6,3 +6,11 @@ edition = "2021"
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
futures = { version = "0.3"}
|
||||
opendal = "0.6"
|
||||
tokio = { version = "1.0", features = ["full"] }
|
||||
|
||||
[dev-dependencies]
|
||||
anyhow = "1.0"
|
||||
common-telemetry = { path = "../common/telemetry"}
|
||||
tempdir = "0.3"
|
||||
|
||||
4
src/object-store/src/backend.rs
Normal file
4
src/object-store/src/backend.rs
Normal file
@@ -0,0 +1,4 @@
|
||||
pub mod azblob;
|
||||
pub mod fs;
|
||||
pub mod memory;
|
||||
pub mod s3;
|
||||
1
src/object-store/src/backend/azblob.rs
Normal file
1
src/object-store/src/backend/azblob.rs
Normal file
@@ -0,0 +1 @@
|
||||
pub use opendal::services::azblob::{Backend, Builder};
|
||||
1
src/object-store/src/backend/fs.rs
Normal file
1
src/object-store/src/backend/fs.rs
Normal file
@@ -0,0 +1 @@
|
||||
pub use opendal::services::fs::{Backend, Builder};
|
||||
1
src/object-store/src/backend/memory.rs
Normal file
1
src/object-store/src/backend/memory.rs
Normal file
@@ -0,0 +1 @@
|
||||
pub use opendal::services::memory::{Backend, Builder};
|
||||
1
src/object-store/src/backend/s3.rs
Normal file
1
src/object-store/src/backend/s3.rs
Normal file
@@ -0,0 +1 @@
|
||||
pub use opendal::services::s3::{Backend, Builder};
|
||||
@@ -1 +1,5 @@
|
||||
|
||||
pub use opendal::{
|
||||
Accessor, Layer, Metadata, Object, ObjectMode, ObjectStreamer, Operator as ObjectStore,
|
||||
};
|
||||
pub mod backend;
|
||||
pub mod util;
|
||||
|
||||
7
src/object-store/src/util.rs
Normal file
7
src/object-store/src/util.rs
Normal file
@@ -0,0 +1,7 @@
|
||||
use futures::TryStreamExt;
|
||||
|
||||
use crate::{Object, ObjectStreamer};
|
||||
|
||||
pub async fn collect(stream: ObjectStreamer) -> Result<Vec<Object>, std::io::Error> {
|
||||
stream.try_collect::<Vec<_>>().await
|
||||
}
|
||||
112
src/object-store/tests/object_store_test.rs
Normal file
112
src/object-store/tests/object_store_test.rs
Normal file
@@ -0,0 +1,112 @@
|
||||
use std::env;
|
||||
|
||||
use anyhow::Result;
|
||||
use common_telemetry::logging;
|
||||
use object_store::{
|
||||
backend::{fs, s3},
|
||||
util, Object, ObjectMode, ObjectStore, ObjectStreamer,
|
||||
};
|
||||
use tempdir::TempDir;
|
||||
|
||||
async fn test_object_crud(store: &ObjectStore) -> Result<()> {
|
||||
// Create object handler.
|
||||
let object = store.object("test_file");
|
||||
|
||||
// Write data info object;
|
||||
assert!(object.write("Hello, World!").await.is_ok());
|
||||
|
||||
// Read data from object;
|
||||
let bs = object.read().await?;
|
||||
assert_eq!("Hello, World!", String::from_utf8(bs)?);
|
||||
|
||||
// Read range from object;
|
||||
let bs = object.range_read(1..=11).await?;
|
||||
assert_eq!("ello, World", String::from_utf8(bs)?);
|
||||
|
||||
// Get object's Metadata
|
||||
let meta = object.metadata().await?;
|
||||
assert!(meta.complete());
|
||||
assert_eq!("test_file", meta.path());
|
||||
assert_eq!(ObjectMode::FILE, meta.mode());
|
||||
assert_eq!(13, meta.content_length());
|
||||
|
||||
// Delete object.
|
||||
assert!(object.delete().await.is_ok());
|
||||
assert!(object.read().await.is_err());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn test_object_list(store: &ObjectStore) -> Result<()> {
|
||||
// Create some object handlers.
|
||||
let o1 = store.object("test_file1");
|
||||
let o2 = store.object("test_file2");
|
||||
let o3 = store.object("test_file3");
|
||||
|
||||
// Write something
|
||||
assert!(o1.write("Hello, object1!").await.is_ok());
|
||||
assert!(o2.write("Hello, object2!").await.is_ok());
|
||||
assert!(o3.write("Hello, object3!").await.is_ok());
|
||||
|
||||
// List objects
|
||||
let o: Object = store.object("/");
|
||||
let obs: ObjectStreamer = o.list().await?;
|
||||
let objects = util::collect(obs).await?;
|
||||
assert_eq!(3, objects.len());
|
||||
|
||||
// Delete o1, o3
|
||||
assert!(o1.delete().await.is_ok());
|
||||
assert!(o3.delete().await.is_ok());
|
||||
|
||||
// List obejcts again
|
||||
let objects = util::collect(o.list().await?).await?;
|
||||
assert_eq!(1, objects.len());
|
||||
|
||||
// Only o2 is exists
|
||||
let o2 = &objects[0];
|
||||
let bs = o2.read().await?;
|
||||
assert_eq!("Hello, object2!", String::from_utf8(bs)?);
|
||||
// Delete o2
|
||||
assert!(o2.delete().await.is_ok());
|
||||
|
||||
let objects = util::collect(o.list().await?).await?;
|
||||
assert!(objects.is_empty());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_fs_backend() -> Result<()> {
|
||||
let tmp_dir = TempDir::new("test_fs_backend")?;
|
||||
let store = ObjectStore::new(
|
||||
fs::Backend::build()
|
||||
.root(&tmp_dir.path().to_string_lossy())
|
||||
.finish()
|
||||
.await?,
|
||||
);
|
||||
|
||||
test_object_crud(&store).await?;
|
||||
test_object_list(&store).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_s3_backend() -> Result<()> {
|
||||
logging::init_default_ut_logging();
|
||||
if env::var("GT_S3_BUCKET").is_ok() {
|
||||
logging::info!("Running s3 test.");
|
||||
let store = ObjectStore::new(
|
||||
s3::Backend::build()
|
||||
.access_key_id(&env::var("GT_S3_ACCESS_KEY_ID")?)
|
||||
.secret_access_key(&env::var("GT_S3_ACCESS_KEY")?)
|
||||
.bucket(&env::var("GT_S3_BUCKET")?)
|
||||
.finish()
|
||||
.await?,
|
||||
);
|
||||
test_object_crud(&store).await?;
|
||||
test_object_list(&store).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user