From a819b6bce5bcda97e5ab5a9e9b11042275b4f1ac Mon Sep 17 00:00:00 2001
From: dennis zhuang <killme2008@gmail.com>
Date: Tue, 17 May 2022 15:58:27 +0800
Subject: [PATCH] feat: impl object-store based on opendal (#27)

* feat: impl object-store based on opendal

* feat: adds s3 test

* feat: export Layer

* feat: pass s3 access key/secret by env vars

* fix: logging when s3 test is running
---
 .github/workflows/coverage.yml              |   3 +
 .github/workflows/develop.yml               |   4 +
 Cargo.lock                                  | 436 +++++++++++++++++++-
 src/object-store/Cargo.toml                 |   8 +
 src/object-store/src/backend.rs             |   4 +
 src/object-store/src/backend/azblob.rs      |   1 +
 src/object-store/src/backend/fs.rs          |   1 +
 src/object-store/src/backend/memory.rs      |   1 +
 src/object-store/src/backend/s3.rs          |   1 +
 src/object-store/src/lib.rs                 |   6 +-
 src/object-store/src/util.rs                |   7 +
 src/object-store/tests/object_store_test.rs | 112 +++++
 12 files changed, 577 insertions(+), 7 deletions(-)
 create mode 100644 src/object-store/src/backend.rs
 create mode 100644 src/object-store/src/backend/azblob.rs
 create mode 100644 src/object-store/src/backend/fs.rs
 create mode 100644 src/object-store/src/backend/memory.rs
 create mode 100644 src/object-store/src/backend/s3.rs
 create mode 100644 src/object-store/src/util.rs
 create mode 100644 src/object-store/tests/object_store_test.rs

diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
index 01abd0e8a9..d1f952eee4 100644
--- a/.github/workflows/coverage.yml
+++ b/.github/workflows/coverage.yml
@@ -29,6 +29,9 @@ jobs:
         env:
           CARGO_INCREMENTAL: 0
           RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=abort -Zpanic_abort_tests"
+          GT_S3_BUCKET: ${{ secrets.S3_BUCKET }}
+          GT_S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
+          GT_S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
       - name: Gather coverage data
         id: coverage
         uses: actions-rs/grcov@v0.1
diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml
index 032bfb9fd6..a2a3505cf4 100644
--- a/.github/workflows/develop.yml
+++ b/.github/workflows/develop.yml
@@ -35,6 +35,10 @@ jobs:
         with:
           command: test
           args: --workspace
+        env:
+          GT_S3_BUCKET: ${{ secrets.S3_BUCKET }}
+          GT_S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
+          GT_S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
 
   fmt:
     name: Rustfmt
diff --git a/Cargo.lock b/Cargo.lock
index 5610768767..430fd5dd31 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -127,6 +127,19 @@ dependencies = [
  "strength_reduce",
 ]
 
+[[package]]
+name = "async-compat"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b48b4ff0c2026db683dea961cd8ea874737f56cffca86fa84415eaddc51c00d"
+dependencies = [
+ "futures-core",
+ "futures-io",
+ "once_cell",
+ "pin-project-lite",
+ "tokio",
+]
+
 [[package]]
 name = "async-compression"
 version = "0.3.12"
@@ -632,6 +645,20 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "crossbeam"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ae5588f6b3c3cb05239e90bd110f257254aecd01e4635400391aeae07497845"
+dependencies = [
+ "cfg-if",
+ "crossbeam-channel",
+ "crossbeam-deque",
+ "crossbeam-epoch",
+ "crossbeam-queue",
+ "crossbeam-utils",
+]
+
 [[package]]
 name = "crossbeam-channel"
 version = "0.5.4"
@@ -642,6 +669,17 @@ dependencies = [
  "crossbeam-utils",
 ]
 
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e"
+dependencies = [
+ "cfg-if",
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
 [[package]]
 name = "crossbeam-epoch"
 version = "0.9.8"
@@ -656,6 +694,16 @@ dependencies = [
  "scopeguard",
 ]
 
+[[package]]
+name = "crossbeam-queue"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1f25d8400f4a7a5778f0e4e52384a48cbd9b5c495d110786187fc750075277a2"
+dependencies = [
+ "cfg-if",
+ "crossbeam-utils",
+]
+
 [[package]]
 name = "crossbeam-utils"
 version = "0.8.8"
@@ -704,6 +752,16 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "ctor"
+version = "0.1.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f877be4f7c9f246b183111634f75baa039715e3f46ce860677d3b19a69fb229c"
+dependencies = [
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "datafusion"
 version = "7.0.0"
@@ -727,7 +785,7 @@ dependencies = [
  "parquet2",
  "paste",
  "pin-project-lite",
- "rand",
+ "rand 0.8.5",
  "smallvec",
  "sqlparser",
  "tempfile",
@@ -774,7 +832,7 @@ dependencies = [
  "md-5",
  "ordered-float 2.10.0",
  "paste",
- "rand",
+ "rand 0.8.5",
  "regex",
  "sha2",
  "unicode-segmentation",
@@ -827,6 +885,32 @@ dependencies = [
  "subtle",
 ]
 
+[[package]]
+name = "dirs"
+version = "4.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059"
+dependencies = [
+ "dirs-sys",
+]
+
+[[package]]
+name = "dirs-sys"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6"
+dependencies = [
+ "libc",
+ "redox_users",
+ "winapi",
+]
+
+[[package]]
+name = "dlv-list"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257"
+
 [[package]]
 name = "doc-comment"
 version = "0.3.3"
@@ -912,6 +996,12 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "fuchsia-cprng"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
+
 [[package]]
 name = "futures"
 version = "0.3.21"
@@ -1119,6 +1209,21 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "hex"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
+
+[[package]]
+name = "hmac"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
+dependencies = [
+ "digest",
+]
+
 [[package]]
 name = "http"
 version = "0.2.7"
@@ -1308,6 +1413,20 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "jsonwebtoken"
+version = "8.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc9051c17f81bae79440afa041b3a278e1de71bfb96d32454b477fd4703ccb6f"
+dependencies = [
+ "base64",
+ "pem",
+ "ring",
+ "serde",
+ "serde_json",
+ "simple_asn1",
+]
+
 [[package]]
 name = "lazy_static"
 version = "1.4.0"
@@ -1470,6 +1589,12 @@ dependencies = [
  "digest",
 ]
 
+[[package]]
+name = "md5"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
+
 [[package]]
 name = "memchr"
 version = "2.5.0"
@@ -1559,6 +1684,34 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
 
+[[package]]
+name = "minitrace"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a07fdf302cc0591c97eb45939550ddaddd9962e400c20b319aa16c244cb1f16"
+dependencies = [
+ "crossbeam",
+ "futures",
+ "minitrace-macro",
+ "minstant",
+ "once_cell",
+ "parking_lot 0.11.2",
+ "pin-project",
+ "retain_mut",
+]
+
+[[package]]
+name = "minitrace-macro"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4132dfe6097f4a90c0bbb34be0687c38d14303dd2e74f8442ae80e9bc5a34c47"
+dependencies = [
+ "proc-macro-error",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "miniz_oxide"
 version = "0.5.1"
@@ -1568,6 +1721,17 @@ dependencies = [
  "adler",
 ]
 
+[[package]]
+name = "minstant"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2cb320648b7883b43ce5dfbc5c6f4a84038194c3f67b4fcb7d05c994e6006557"
+dependencies = [
+ "ctor",
+ "libc",
+ "wasi 0.7.0",
+]
+
 [[package]]
 name = "mio"
 version = "0.8.3"
@@ -1628,6 +1792,17 @@ dependencies = [
  "minimal-lexical",
 ]
 
+[[package]]
+name = "num-bigint"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f"
+dependencies = [
+ "autocfg",
+ "num-integer",
+ "num-traits",
+]
+
 [[package]]
 name = "num-integer"
 version = "0.1.45"
@@ -1678,6 +1853,14 @@ dependencies = [
 [[package]]
 name = "object-store"
 version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "common-telemetry",
+ "futures",
+ "opendal",
+ "tempdir",
+ "tokio",
+]
 
 [[package]]
 name = "once_cell"
@@ -1685,6 +1868,38 @@ version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9"
 
+[[package]]
+name = "opendal"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3649ace5a99d388ac9d02459135ad0425941e8cf6c33f418c4ded80483563ce3"
+dependencies = [
+ "anyhow",
+ "async-compat",
+ "async-trait",
+ "base64",
+ "bytes",
+ "futures",
+ "http",
+ "hyper",
+ "hyper-tls",
+ "log",
+ "md5",
+ "metrics",
+ "minitrace",
+ "once_cell",
+ "parking_lot 0.12.0",
+ "pin-project",
+ "quick-xml",
+ "reqsign",
+ "reqwest",
+ "roxmltree",
+ "serde",
+ "thiserror",
+ "time 0.3.9",
+ "tokio",
+]
+
 [[package]]
 name = "openssl"
 version = "0.10.40"
@@ -1745,7 +1960,7 @@ dependencies = [
  "lazy_static",
  "percent-encoding",
  "pin-project",
- "rand",
+ "rand 0.8.5",
  "thiserror",
  "tokio",
  "tokio-stream",
@@ -1793,6 +2008,16 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "ordered-multimap"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a"
+dependencies = [
+ "dlv-list",
+ "hashbrown 0.12.1",
+]
+
 [[package]]
 name = "os_str_bytes"
 version = "6.0.0"
@@ -1887,6 +2112,15 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0c520e05135d6e763148b6426a837e239041653ba7becd2e538c076c738025fc"
 
+[[package]]
+name = "pem"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e9a3b09a20e374558580a4914d3b7d89bd61b954a5a5e1dcbea98753addb1947"
+dependencies = [
+ "base64",
+]
+
 [[package]]
 name = "percent-encoding"
 version = "2.1.0"
@@ -2059,6 +2293,25 @@ dependencies = [
  "tokio-stream",
 ]
 
+[[package]]
+name = "quick-xml"
+version = "0.22.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8533f14c8382aaad0d592c812ac3b826162128b65662331e1127b45c3d18536b"
+dependencies = [
+ "memchr",
+ "serde",
+]
+
+[[package]]
+name = "quickcheck"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6"
+dependencies = [
+ "rand 0.8.5",
+]
+
 [[package]]
 name = "quote"
 version = "1.0.18"
@@ -2068,6 +2321,19 @@ dependencies = [
  "proc-macro2",
 ]
 
+[[package]]
+name = "rand"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293"
+dependencies = [
+ "fuchsia-cprng",
+ "libc",
+ "rand_core 0.3.1",
+ "rdrand",
+ "winapi",
+]
+
 [[package]]
 name = "rand"
 version = "0.8.5"
@@ -2076,7 +2342,7 @@ checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
 dependencies = [
  "libc",
  "rand_chacha",
- "rand_core",
+ "rand_core 0.6.3",
 ]
 
 [[package]]
@@ -2086,9 +2352,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
 dependencies = [
  "ppv-lite86",
- "rand_core",
+ "rand_core 0.6.3",
 ]
 
+[[package]]
+name = "rand_core"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b"
+dependencies = [
+ "rand_core 0.4.2",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc"
+
 [[package]]
 name = "rand_core"
 version = "0.6.3"
@@ -2107,6 +2388,15 @@ dependencies = [
  "bitflags",
 ]
 
+[[package]]
+name = "rdrand"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2"
+dependencies = [
+ "rand_core 0.3.1",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.2.13"
@@ -2116,6 +2406,17 @@ dependencies = [
  "bitflags",
 ]
 
+[[package]]
+name = "redox_users"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
+dependencies = [
+ "getrandom",
+ "redox_syscall",
+ "thiserror",
+]
+
 [[package]]
 name = "regex"
 version = "1.5.5"
@@ -2151,6 +2452,35 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "reqsign"
+version = "0.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8931679eac96ffc8eee4e5507c4b91fbc0799f29a6535707ee3ef89c0d0de426"
+dependencies = [
+ "anyhow",
+ "async-trait",
+ "base64",
+ "bytes",
+ "dirs",
+ "form_urlencoded",
+ "hex",
+ "hmac",
+ "http",
+ "jsonwebtoken",
+ "log",
+ "once_cell",
+ "percent-encoding",
+ "reqwest",
+ "roxmltree",
+ "rust-ini",
+ "serde",
+ "serde_json",
+ "sha2",
+ "time 0.3.9",
+ "tokio",
+]
+
 [[package]]
 name = "reqwest"
 version = "0.11.10"
@@ -2189,6 +2519,46 @@ dependencies = [
  "winreg",
 ]
 
+[[package]]
+name = "retain_mut"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8c31b5c4033f8fdde8700e4657be2c497e7288f01515be52168c631e2e4d4086"
+
+[[package]]
+name = "ring"
+version = "0.16.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc"
+dependencies = [
+ "cc",
+ "libc",
+ "once_cell",
+ "spin",
+ "untrusted",
+ "web-sys",
+ "winapi",
+]
+
+[[package]]
+name = "roxmltree"
+version = "0.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "921904a62e410e37e215c40381b7117f830d9d89ba60ab5236170541dd25646b"
+dependencies = [
+ "xmlparser",
+]
+
+[[package]]
+name = "rust-ini"
+version = "0.18.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df"
+dependencies = [
+ "cfg-if",
+ "ordered-multimap",
+]
+
 [[package]]
 name = "rustc-demangle"
 version = "0.1.21"
@@ -2325,6 +2695,18 @@ version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a"
 
+[[package]]
+name = "simple_asn1"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4a762b1c38b9b990c694b9c2f8abe3372ce6a9ceaae6bca39cfc46e054f45745"
+dependencies = [
+ "num-bigint",
+ "num-traits",
+ "thiserror",
+ "time 0.3.9",
+]
+
 [[package]]
 name = "sketches-ddsketch"
 version = "0.1.2"
@@ -2382,6 +2764,12 @@ dependencies = [
  "windows-sys",
 ]
 
+[[package]]
+name = "spin"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
+
 [[package]]
 name = "sql"
 version = "0.1.0"
@@ -2493,6 +2881,16 @@ dependencies = [
  "snafu",
 ]
 
+[[package]]
+name = "tempdir"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8"
+dependencies = [
+ "rand 0.4.6",
+ "remove_dir_all",
+]
+
 [[package]]
 name = "tempfile"
 version = "3.3.0"
@@ -2603,8 +3001,16 @@ dependencies = [
  "itoa 1.0.1",
  "libc",
  "num_threads",
+ "quickcheck",
+ "time-macros",
 ]
 
+[[package]]
+name = "time-macros"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792"
+
 [[package]]
 name = "tinyvec"
 version = "1.6.0"
@@ -2755,7 +3161,7 @@ dependencies = [
  "indexmap",
  "pin-project",
  "pin-project-lite",
- "rand",
+ "rand 0.8.5",
  "slab",
  "tokio",
  "tokio-util 0.7.1",
@@ -2976,6 +3382,12 @@ version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04"
 
+[[package]]
+name = "untrusted"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
+
 [[package]]
 name = "url"
 version = "2.2.2"
@@ -3025,6 +3437,12 @@ dependencies = [
  "try-lock",
 ]
 
+[[package]]
+name = "wasi"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b89c3ce4ce14bdc6fb6beaf9ec7928ca331de5df7e5ea278375642a2f478570d"
+
 [[package]]
 name = "wasi"
 version = "0.10.2+wasi-snapshot-preview1"
@@ -3196,6 +3614,12 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "xmlparser"
+version = "0.13.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "114ba2b24d2167ef6d67d7d04c8cc86522b87f490025f39f0303b7db5bf5e3d8"
+
 [[package]]
 name = "zstd"
 version = "0.10.0+zstd.1.5.2"
diff --git a/src/object-store/Cargo.toml b/src/object-store/Cargo.toml
index 00608ff005..0ccf67b137 100644
--- a/src/object-store/Cargo.toml
+++ b/src/object-store/Cargo.toml
@@ -6,3 +6,11 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+futures = { version = "0.3"}
+opendal = "0.6"
+tokio = { version = "1.0", features = ["full"] }
+
+[dev-dependencies]
+anyhow = "1.0"
+common-telemetry = { path = "../common/telemetry"}
+tempdir = "0.3"
diff --git a/src/object-store/src/backend.rs b/src/object-store/src/backend.rs
new file mode 100644
index 0000000000..b28002fe5e
--- /dev/null
+++ b/src/object-store/src/backend.rs
@@ -0,0 +1,4 @@
+pub mod azblob;
+pub mod fs;
+pub mod memory;
+pub mod s3;
diff --git a/src/object-store/src/backend/azblob.rs b/src/object-store/src/backend/azblob.rs
new file mode 100644
index 0000000000..0296e6ed2c
--- /dev/null
+++ b/src/object-store/src/backend/azblob.rs
@@ -0,0 +1 @@
+pub use opendal::services::azblob::{Backend, Builder};
diff --git a/src/object-store/src/backend/fs.rs b/src/object-store/src/backend/fs.rs
new file mode 100644
index 0000000000..b8f727ed58
--- /dev/null
+++ b/src/object-store/src/backend/fs.rs
@@ -0,0 +1 @@
+pub use opendal::services::fs::{Backend, Builder};
diff --git a/src/object-store/src/backend/memory.rs b/src/object-store/src/backend/memory.rs
new file mode 100644
index 0000000000..3a27615e12
--- /dev/null
+++ b/src/object-store/src/backend/memory.rs
@@ -0,0 +1 @@
+pub use opendal::services::memory::{Backend, Builder};
diff --git a/src/object-store/src/backend/s3.rs b/src/object-store/src/backend/s3.rs
new file mode 100644
index 0000000000..8eec123285
--- /dev/null
+++ b/src/object-store/src/backend/s3.rs
@@ -0,0 +1 @@
+pub use opendal::services::s3::{Backend, Builder};
diff --git a/src/object-store/src/lib.rs b/src/object-store/src/lib.rs
index 8b13789179..5043c2f561 100644
--- a/src/object-store/src/lib.rs
+++ b/src/object-store/src/lib.rs
@@ -1 +1,5 @@
-
+pub use opendal::{
+    Accessor, Layer, Metadata, Object, ObjectMode, ObjectStreamer, Operator as ObjectStore,
+};
+pub mod backend;
+pub mod util;
diff --git a/src/object-store/src/util.rs b/src/object-store/src/util.rs
new file mode 100644
index 0000000000..93b8c0f801
--- /dev/null
+++ b/src/object-store/src/util.rs
@@ -0,0 +1,7 @@
+use futures::TryStreamExt;
+
+use crate::{Object, ObjectStreamer};
+
+pub async fn collect(stream: ObjectStreamer) -> Result<Vec<Object>, std::io::Error> {
+    stream.try_collect::<Vec<_>>().await
+}
diff --git a/src/object-store/tests/object_store_test.rs b/src/object-store/tests/object_store_test.rs
new file mode 100644
index 0000000000..c51eb8219b
--- /dev/null
+++ b/src/object-store/tests/object_store_test.rs
@@ -0,0 +1,112 @@
+use std::env;
+
+use anyhow::Result;
+use common_telemetry::logging;
+use object_store::{
+    backend::{fs, s3},
+    util, Object, ObjectMode, ObjectStore, ObjectStreamer,
+};
+use tempdir::TempDir;
+
+async fn test_object_crud(store: &ObjectStore) -> Result<()> {
+    // Create object handler.
+    let object = store.object("test_file");
+
+    // Write data info object;
+    assert!(object.write("Hello, World!").await.is_ok());
+
+    // Read data from object;
+    let bs = object.read().await?;
+    assert_eq!("Hello, World!", String::from_utf8(bs)?);
+
+    // Read range from object;
+    let bs = object.range_read(1..=11).await?;
+    assert_eq!("ello, World", String::from_utf8(bs)?);
+
+    // Get object's Metadata
+    let meta = object.metadata().await?;
+    assert!(meta.complete());
+    assert_eq!("test_file", meta.path());
+    assert_eq!(ObjectMode::FILE, meta.mode());
+    assert_eq!(13, meta.content_length());
+
+    // Delete object.
+    assert!(object.delete().await.is_ok());
+    assert!(object.read().await.is_err());
+
+    Ok(())
+}
+
+async fn test_object_list(store: &ObjectStore) -> Result<()> {
+    // Create  some object handlers.
+    let o1 = store.object("test_file1");
+    let o2 = store.object("test_file2");
+    let o3 = store.object("test_file3");
+
+    // Write something
+    assert!(o1.write("Hello, object1!").await.is_ok());
+    assert!(o2.write("Hello, object2!").await.is_ok());
+    assert!(o3.write("Hello, object3!").await.is_ok());
+
+    // List objects
+    let o: Object = store.object("/");
+    let obs: ObjectStreamer = o.list().await?;
+    let objects = util::collect(obs).await?;
+    assert_eq!(3, objects.len());
+
+    // Delete o1, o3
+    assert!(o1.delete().await.is_ok());
+    assert!(o3.delete().await.is_ok());
+
+    // List obejcts again
+    let objects = util::collect(o.list().await?).await?;
+    assert_eq!(1, objects.len());
+
+    // Only o2 is exists
+    let o2 = &objects[0];
+    let bs = o2.read().await?;
+    assert_eq!("Hello, object2!", String::from_utf8(bs)?);
+    // Delete o2
+    assert!(o2.delete().await.is_ok());
+
+    let objects = util::collect(o.list().await?).await?;
+    assert!(objects.is_empty());
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_fs_backend() -> Result<()> {
+    let tmp_dir = TempDir::new("test_fs_backend")?;
+    let store = ObjectStore::new(
+        fs::Backend::build()
+            .root(&tmp_dir.path().to_string_lossy())
+            .finish()
+            .await?,
+    );
+
+    test_object_crud(&store).await?;
+    test_object_list(&store).await?;
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_s3_backend() -> Result<()> {
+    logging::init_default_ut_logging();
+    if env::var("GT_S3_BUCKET").is_ok() {
+        logging::info!("Running s3 test.");
+        let store = ObjectStore::new(
+            s3::Backend::build()
+                .access_key_id(&env::var("GT_S3_ACCESS_KEY_ID")?)
+                .secret_access_key(&env::var("GT_S3_ACCESS_KEY")?)
+                .bucket(&env::var("GT_S3_BUCKET")?)
+                .finish()
+                .await?,
+        );
+        test_object_crud(&store).await?;
+        test_object_list(&store).await?;
+    }
+
+    Ok(())
+}