From f24364997132eb1fb07e1cff022f84ff80bee6aa Mon Sep 17 00:00:00 2001 From: Ning Sun Date: Mon, 17 Oct 2022 19:29:17 +0800 Subject: [PATCH] refactor: Removed openssl from build requirement (#308) * refactor:replace another axum-test-helper branch * refactor: upgrade opendal version * refactor: use cursor for file buffer * refactor:remove native-tls in mysql_async * refactor: use async block and pipeline for newer opendal api * chore: update Cargo.lock * chore: update dependencies * docs: removed openssl from build requirement * fix: call close on pipe writer to flush reader for parquet streamer * refactor: remove redundant return * chore: use pinned revision for our forked mysql_async * style: avoid wild-card import in test code * Apply suggestions from code review Co-authored-by: Yingwen * style: use chained call for builder Co-authored-by: liangxingjian <965662709@qq.com> Co-authored-by: Yingwen --- Cargo.lock | 647 +++++++------------- README.md | 18 - src/datanode/Cargo.toml | 2 +- src/datanode/src/instance.rs | 7 +- src/datanode/src/sql.rs | 4 +- src/object-store/Cargo.toml | 2 +- src/object-store/src/lib.rs | 2 +- src/object-store/tests/object_store_test.rs | 21 +- src/servers/Cargo.toml | 4 +- src/storage/Cargo.toml | 1 + src/storage/src/engine.rs | 5 +- src/storage/src/manifest/region.rs | 5 +- src/storage/src/manifest/storage.rs | 14 +- src/storage/src/region/tests.rs | 5 +- src/storage/src/sst/parquet.rs | 81 +-- src/storage/src/test_util/config_util.rs | 4 +- src/table-engine/src/table/test_util.rs | 5 +- 17 files changed, 316 insertions(+), 511 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1a763824a2..ccde0f7d14 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -182,9 +182,9 @@ dependencies = [ [[package]] name = "ascii" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbf56136a5198c7b01a49e3afcbef6cf84597273d298f54432926024107b0109" +checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" [[package]] name = "async-channel" @@ -388,8 +388,7 @@ dependencies = [ [[package]] name = "axum-test-helper" version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5f0c689f3a3cb707ea097813153343b74dcf73b3e46dedb25be91a24050913" +source = "git+https://github.com/sunng87/axum-test-helper.git?branch=patch-1#c90b5fed699080636330f3a97c1ee20d845329f0" dependencies = [ "axum 0.5.16", "bytes", @@ -405,9 +404,9 @@ dependencies = [ [[package]] name = "backon" -version = "0.0.2" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f334d8b7d003e7d4e17844b81ffbfcd24ad955777997440701c08a834e407105" +checksum = "a62fde477410f4e8a92e2fd4b8536b16ad98270478211d1516cded50bad7208e" dependencies = [ "futures", "pin-project", @@ -630,12 +629,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" -[[package]] -name = "castaway" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2698f953def977c68f935bb0dfa959375ad4638570e969e2f1e9f433cbf1af6" - [[package]] name = "catalog" version = "0.1.0" @@ -721,10 +714,16 @@ dependencies = [ ] [[package]] -name = "clang-sys" -version = "1.3.3" +name = "chunked_transfer" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a050e2153c5be08febd6734e29298e844fdb0fa21aeddd63b4eb7baa106c69b" +checksum = "fff857943da45f546682664a79488be82e69e43c1a7a2307679ab9afb3a66d2e" + +[[package]] +name = "clang-sys" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa2e27ae6ab525c3d369ded447057bca5438d86dc3a68f6faafb8269ba82ebf3" dependencies = [ "glob", "libc", @@ -748,9 +747,9 @@ dependencies = [ [[package]] name = "clap" -version = "3.2.21" +version = "3.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ed5341b2301a26ab80be5cbdced622e80ed808483c52e45e3310a877d3b37d7" +checksum = "86447ad904c7fb335a790c9d7fe3d0d971dc523b8ccd1561a520de9a85302750" dependencies = [ "atty", "bitflags", @@ -760,7 +759,7 @@ dependencies = [ "once_cell", "strsim 0.10.0", "termcolor", - "textwrap 0.15.0", + "textwrap 0.15.1", ] [[package]] @@ -832,7 +831,7 @@ dependencies = [ name = "cmd" version = "0.1.0" dependencies = [ - "clap 3.2.21", + "clap 3.2.22", "common-error", "common-telemetry", "datanode", @@ -964,7 +963,7 @@ version = "0.1.0" dependencies = [ "common-error", "common-telemetry", - "metrics 0.20.1", + "metrics", "once_cell", "paste", "snafu", @@ -979,12 +978,12 @@ dependencies = [ "backtrace", "common-error", "console-subscriber", - "metrics 0.20.1", + "metrics", "metrics-exporter-prometheus", "once_cell", "opentelemetry", "opentelemetry-jaeger", - "parking_lot 0.12.1", + "parking_lot", "tracing", "tracing-appender", "tracing-bunyan-formatter", @@ -1262,47 +1261,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "ctor" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdffe87e1d521a10f9696f833fe502293ea446d7f256c06128293a4119bdf4cb" -dependencies = [ - "quote", - "syn", -] - -[[package]] -name = "curl" -version = "0.4.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "509bd11746c7ac09ebd19f0b17782eae80aadee26237658a6b4808afb5c11a22" -dependencies = [ - "curl-sys", - "libc", - "openssl-probe", - "openssl-sys", - "schannel", - "socket2", - "winapi", -] - -[[package]] -name = "curl-sys" -version = "0.4.56+curl-7.83.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6093e169dd4de29e468fa649fbae11cdcd5551c81fe5bf1b0677adad7ef3d26f" -dependencies = [ - "cc", - "libc", - "libnghttp2-sys", - "libz-sys", - "openssl-sys", - "pkg-config", - "vcpkg", - "winapi", -] - [[package]] name = "darling" version = "0.14.1" @@ -1357,7 +1315,7 @@ dependencies = [ "log", "num_cpus", "ordered-float 2.10.0", - "parking_lot 0.12.1", + "parking_lot", "parquet2", "paste", "pin-project-lite", @@ -1440,7 +1398,7 @@ dependencies = [ "futures", "hyper", "log-store", - "metrics 0.20.1", + "metrics", "object-store", "query", "script", @@ -1473,7 +1431,7 @@ dependencies = [ "enum_dispatch", "num", "num-traits", - "ordered-float 3.0.0", + "ordered-float 3.1.0", "paste", "serde", "serde_json", @@ -1524,9 +1482,9 @@ dependencies = [ [[package]] name = "digest" -version = "0.10.3" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506" +checksum = "adfbc57365a37acbd2ebf2b64d7e69bb766e2fea813521ed536f5d0520dcf86c" dependencies = [ "block-buffer", "crypto-common", @@ -1706,6 +1664,12 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "flagset" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda653ca797810c02f7ca4b804b40b8b95ae046eb989d356bce17919a8c25499" + [[package]] name = "flate2" version = "1.0.24" @@ -1723,21 +1687,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.1.0" @@ -1913,21 +1862,6 @@ version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbf4d2a7a308fd4578637c0b17c7e1c7ba127b8f6ba00b29f717e9655d85eb68" -[[package]] -name = "futures-lite" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48" -dependencies = [ - "fastrand", - "futures-core", - "futures-io", - "memchr", - "parking", - "pin-project-lite", - "waker-fn", -] - [[package]] name = "futures-macro" version = "0.3.24" @@ -2068,9 +2002,9 @@ dependencies = [ [[package]] name = "hdrhistogram" -version = "7.5.1" +version = "7.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea9fe3952d32674a14e0975009a3547af9ea364995b5ec1add2e23c2ae523ab" +checksum = "7f19b9f54f7c7f55e31401bb647626ce0cf0f67b0004982ce815b3ee72a02aa8" dependencies = [ "base64", "byteorder", @@ -2194,6 +2128,19 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d87c48c02e0dc5e3b849a2041db3029fd066650f8f717c07bf8ed78ccb895cac" +dependencies = [ + "http", + "hyper", + "rustls", + "tokio", + "tokio-rustls", +] + [[package]] name = "hyper-timeout" version = "0.4.1" @@ -2206,29 +2153,15 @@ dependencies = [ "tokio-io-timeout", ] -[[package]] -name = "hyper-tls" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" -dependencies = [ - "bytes", - "hyper", - "native-tls", - "tokio", - "tokio-native-tls", -] - [[package]] name = "iana-time-zone" -version = "0.1.48" +version = "0.1.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "237a0714f28b1ee39ccec0770ccb544eb02c9ef2c82bb096230eefcffa6468b0" +checksum = "3bbaead50122b06e9a973ac20bc7445074d99ad9a0a0654934876908a9cec82c" dependencies = [ "android_system_properties", "core-foundation-sys", "js-sys", - "once_cell", "wasm-bindgen", "winapi", ] @@ -2323,38 +2256,11 @@ dependencies = [ "syn", ] -[[package]] -name = "isahc" -version = "1.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "334e04b4d781f436dc315cb1e7515bd96826426345d498149e4bde36b67f8ee9" -dependencies = [ - "async-channel", - "castaway", - "crossbeam-utils", - "curl", - "curl-sys", - "encoding_rs", - "event-listener", - "futures-lite", - "http", - "log", - "mime", - "once_cell", - "polling", - "slab", - "sluice", - "tracing", - "tracing-futures", - "url", - "waker-fn", -] - [[package]] name = "itertools" -version = "0.10.4" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8bf247779e67a9082a4790b45e71ac7cfd1321331a5c856a74a9faebdab78d0" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" dependencies = [ "either", ] @@ -2496,9 +2402,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.132" +version = "0.2.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8371e4e5341c3a96db127eb2465ac681ced4c433e01dd0e938adbef26ba93ba5" +checksum = "c0f80d65747a3e43d1596c7c5492d95d5edddaabd45a7fcdb02b95f644164966" [[package]] name = "libloading" @@ -2516,16 +2422,6 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "292a948cd991e376cf75541fe5b97a1081d713c618b4f1b9500f8844e49eb565" -[[package]] -name = "libnghttp2-sys" -version = "0.1.7+1.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57ed28aba195b38d5ff02b9170cbff627e336a20925e43b4945390401c5dc93f" -dependencies = [ - "cc", - "libc", -] - [[package]] name = "libz-sys" version = "1.1.8" @@ -2533,7 +2429,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf" dependencies = [ "cc", - "libc", "pkg-config", "vcpkg", ] @@ -2546,9 +2441,9 @@ checksum = "d4d2456c373231a208ad294c33dc5bff30051eafd954cd4caae83a712b12854d" [[package]] name = "lock_api" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f80bf5aacaf25cbfc8210d1cfb718f2bf3b11c4c54e5afe36c236853a8ec390" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" dependencies = [ "autocfg", "scopeguard", @@ -2683,9 +2578,9 @@ dependencies = [ [[package]] name = "md-5" -version = "0.10.4" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66b48670c893079d3c2ed79114e3644b7004df1c361a4e0ad52e2e6940d07c3d" +checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" dependencies = [ "digest", ] @@ -2711,16 +2606,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "metrics" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "142c53885123b68d94108295a09d4afe1a1388ed95b54d5dacd9a454753030f2" -dependencies = [ - "ahash", - "metrics-macros 0.5.1", -] - [[package]] name = "metrics" version = "0.20.1" @@ -2728,7 +2613,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b9b8653cec6897f73b519a43fba5ee3d50f62fe9af80b428accdcc093b4a849" dependencies = [ "ahash", - "metrics-macros 0.6.0", + "metrics-macros", "portable-atomic", ] @@ -2739,25 +2624,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8603921e1f54ef386189335f288441af761e0fc61bcb552168d9cedfe63ebc70" dependencies = [ "indexmap", - "metrics 0.20.1", + "metrics", "metrics-util", - "parking_lot 0.12.1", + "parking_lot", "portable-atomic", "quanta", "thiserror", ] -[[package]] -name = "metrics-macros" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49e30813093f757be5cf21e50389a24dc7dbb22c49f23b7e8f51d69b508a5ffa" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "metrics-macros" version = "0.6.0" @@ -2778,9 +2652,9 @@ dependencies = [ "crossbeam-epoch", "crossbeam-utils", "hashbrown", - "metrics 0.20.1", + "metrics", "num_cpus", - "parking_lot 0.12.1", + "parking_lot", "portable-atomic", "quanta", "sketches-ddsketch", @@ -2808,34 +2682,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" -[[package]] -name = "minitrace" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a07fdf302cc0591c97eb45939550ddaddd9962e400c20b319aa16c244cb1f16" -dependencies = [ - "crossbeam", - "futures", - "minitrace-macro", - "minstant", - "once_cell", - "parking_lot 0.11.2", - "pin-project", - "retain_mut", -] - -[[package]] -name = "minitrace-macro" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4132dfe6097f4a90c0bbb34be0687c38d14303dd2e74f8442ae80e9bc5a34c47" -dependencies = [ - "proc-macro-error", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "miniz_oxide" version = "0.5.4" @@ -2845,17 +2691,6 @@ dependencies = [ "adler", ] -[[package]] -name = "minstant" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc5dcfca9a0725105ac948b84cfeb69c3942814c696326743797215413f854b9" -dependencies = [ - "ctor", - "libc", - "wasi 0.7.0", -] - [[package]] name = "mio" version = "0.8.4" @@ -2897,8 +2732,7 @@ dependencies = [ [[package]] name = "mysql_async" version = "0.30.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "456207bb9636a0fdade67a64cea7bdebe6730c3c16ee5e34f2c481838ee5a39e" +source = "git+https://github.com/Morranto/mysql_async.git?rev=127b538#127b538e20b880e2855204e480fa784d8d08e150" dependencies = [ "bytes", "crossbeam", @@ -2910,20 +2744,23 @@ dependencies = [ "lru", "mio", "mysql_common", - "native-tls", "once_cell", "pem", "percent-encoding", "pin-project", + "rustls", + "rustls-pemfile 0.2.1", "serde", "serde_json", "socket2", "thiserror", "tokio", - "tokio-native-tls", + "tokio-rustls", "tokio-util", "twox-hash", "url", + "webpki", + "webpki-roots", ] [[package]] @@ -2993,24 +2830,6 @@ dependencies = [ "syn", ] -[[package]] -name = "native-tls" -version = "0.2.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9" -dependencies = [ - "lazy_static", - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - [[package]] name = "nibble_vec" version = "0.1.0" @@ -3185,9 +3004,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f7254b99e31cad77da24b08ebf628882739a608578bb1bcdfc1f9c21260d7c0" +checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1" [[package]] name = "oorandom" @@ -3197,33 +3016,33 @@ checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" [[package]] name = "opendal" -version = "0.9.1" +version = "0.17.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9e982034fd0b4f142efba461604f5ccb1fb1f962c4e84c8e44ce369f0e3d1f2" +checksum = "d1ac2fba66b4b574259a4e1c6a1b57b8e649682ed7da167c9cde120c1495f2cd" dependencies = [ "anyhow", "async-compat", "async-trait", "base64", "bytes", + "flagset", "futures", "http", - "hyper", - "hyper-tls", "log", "md5", - "metrics 0.19.0", - "minitrace", "once_cell", - "parking_lot 0.12.1", + "parking_lot", "percent-encoding", "pin-project", "quick-xml", "reqsign", + "reqwest", "serde", + "serde_json", "thiserror", "time 0.3.14", "tokio", + "ureq", ] [[package]] @@ -3240,51 +3059,12 @@ dependencies = [ "tokio", ] -[[package]] -name = "openssl" -version = "0.10.41" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "618febf65336490dfcf20b73f885f5651a0c89c64c2d4a8c3662585a70bf5bd0" -dependencies = [ - "bitflags", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "openssl-probe" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" -[[package]] -name = "openssl-sys" -version = "0.9.75" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5f9bd0c2710541a3cda73d6f9ac4f1b240de4ae261065d309dbe73d9dceb42f" -dependencies = [ - "autocfg", - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "opentelemetry" version = "0.17.0" @@ -3356,9 +3136,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "3.0.0" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96bcbab4bfea7a59c2c0fe47211a1ac4e3e96bea6eb446d704f310bc5c732ae2" +checksum = "98ffdb14730ed2ef599c65810c15b000896e21e8776b512de0db0c3d7335cc2a" dependencies = [ "num-traits", "serde", @@ -3380,23 +3160,6 @@ version = "6.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ff7415e9ae3fff1225851df9e0d9e4e5479f947619774677a63572e55e80eff" -[[package]] -name = "parking" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" - -[[package]] -name = "parking_lot" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" -dependencies = [ - "instant", - "lock_api", - "parking_lot_core 0.8.5", -] - [[package]] name = "parking_lot" version = "0.12.1" @@ -3404,21 +3167,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core 0.9.3", -] - -[[package]] -name = "parking_lot_core" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" -dependencies = [ - "cfg-if", - "instant", - "libc", - "redox_syscall", - "smallvec", - "winapi", + "parking_lot_core", ] [[package]] @@ -3697,20 +3446,6 @@ dependencies = [ "syn", ] -[[package]] -name = "polling" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899b00b9c8ab553c743b3e11e87c5c7d423b2a2de229ba95b24a756344748011" -dependencies = [ - "autocfg", - "cfg-if", - "libc", - "log", - "wepoll-ffi", - "winapi", -] - [[package]] name = "portable-atomic" version = "0.3.15" @@ -3904,7 +3639,7 @@ dependencies = [ "format_num", "futures", "futures-util", - "metrics 0.20.1", + "metrics", "num", "num-traits", "paste", @@ -3924,9 +3659,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.23.1" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11bafc859c6815fbaffbbbf4229ecb767ac913fecb27f9ad4343662e9ef099ea" +checksum = "58e21a144a0ffb5fad7b464babcdab934a325ad69b7c0373bcfef5cbd9799ca9" dependencies = [ "memchr", "serde", @@ -3978,7 +3713,7 @@ checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", "rand_chacha", - "rand_core 0.6.3", + "rand_core 0.6.4", ] [[package]] @@ -3988,7 +3723,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core 0.6.3", + "rand_core 0.6.4", ] [[package]] @@ -4008,9 +3743,9 @@ checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" [[package]] name = "rand_core" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ "getrandom", ] @@ -4130,9 +3865,9 @@ dependencies = [ [[package]] name = "reqsign" -version = "0.1.1" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a6b48d7d1f390bcb0149b4d7a3022f5a927fca173c19413ba17e74936716e39" +checksum = "d6e28b79573bf9e503b514799a5533935fa63bec22d582d4d11cab1eab7a040d" dependencies = [ "anyhow", "backon", @@ -4143,7 +3878,6 @@ dependencies = [ "hex", "hmac", "http", - "isahc", "jsonwebtoken", "log", "once_cell", @@ -4152,15 +3886,17 @@ dependencies = [ "rust-ini", "serde", "serde_json", + "sha1", "sha2", "time 0.3.14", + "ureq", ] [[package]] name = "reqwest" -version = "0.11.11" +version = "0.11.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b75aa69a3f06bbcc66ede33af2af253c6f7a86b1ca0033f60c580a27074fbf92" +checksum = "431949c384f4e2ae07605ccaa56d1d9d2ecdb5cadd4f9577ccfab29f2e5149fc" dependencies = [ "base64", "bytes", @@ -4171,21 +3907,23 @@ dependencies = [ "http", "http-body", "hyper", - "hyper-tls", + "hyper-rustls", "ipnet", "js-sys", - "lazy_static", "log", "mime", "mime_guess", - "native-tls", + "once_cell", "percent-encoding", "pin-project-lite", + "rustls", + "rustls-native-certs", + "rustls-pemfile 1.0.1", "serde", "serde_json", "serde_urlencoded", "tokio", - "tokio-native-tls", + "tokio-rustls", "tokio-util", "tower-service", "url", @@ -4217,12 +3955,6 @@ dependencies = [ "syn-ext", ] -[[package]] -name = "retain_mut" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4389f1d5789befaf6029ebd9f7dac4af7f7e3d61b69d4f30e2ac02b57e7712b0" - [[package]] name = "ring" version = "0.16.20" @@ -4293,9 +4025,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.35.9" +version = "0.35.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72c825b8aa8010eb9ee99b75f05e10180b9278d161583034d7574c9d617aeada" +checksum = "af895b90e5c071badc3136fc10ff0bcfc98747eadbaf43ed8f214e07ba8f8477" dependencies = [ "bitflags", "errno", @@ -4305,6 +4037,48 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "rustls" +version = "0.20.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aab8ee6c7097ed6057f43c187a62418d0c05a4bd5f18b3571db50ee0f9ce033" +dependencies = [ + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "rustls-native-certs" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50" +dependencies = [ + "openssl-probe", + "rustls-pemfile 1.0.1", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eebeaeb360c87bfb72e84abdb3447159c0eaececf1bef2aecd65a8be949d1c9" +dependencies = [ + "base64", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0864aeff53f8c05aa08d86e5ef839d3dfcf07aeba2db32f12db0ef716e87bd55" +dependencies = [ + "base64", +] + [[package]] name = "rustpython-ast" version = "0.1.0" @@ -4395,7 +4169,7 @@ dependencies = [ "rustpython-doc", "syn", "syn-ext", - "textwrap 0.15.0", + "textwrap 0.15.1", ] [[package]] @@ -4468,7 +4242,7 @@ dependencies = [ "num_enum", "once_cell", "optional", - "parking_lot 0.12.1", + "parking_lot", "paste", "rand 0.8.5", "result-like", @@ -4612,6 +4386,16 @@ dependencies = [ "tokio-test", ] +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "security-framework" version = "2.7.0" @@ -4637,15 +4421,15 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f6841e709003d68bb2deee8c343572bf446003ec20a583e76f7b15cebf3711" +checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4" [[package]] name = "serde" -version = "1.0.144" +version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f747710de3dcd43b88c9168773254e809d8ddbdf9653b84e2554ab219f17860" +checksum = "728eb6351430bccb993660dfffc5a72f91ccc1295abaa8ce19b27ebe4f75568b" dependencies = [ "serde_derive", ] @@ -4662,9 +4446,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.144" +version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94ed3a816fb1d101812f83e789f888322c34e291f894f19590dc310963e87a00" +checksum = "81fa1584d3d1bcacd84c277a0dfe21f5b0f6accf4a23d04d4c6d61f1af522b4c" dependencies = [ "proc-macro2", "quote", @@ -4720,7 +4504,7 @@ dependencies = [ "hex", "hyper", "influxdb_line_protocol", - "metrics 0.20.1", + "metrics", "mysql_async", "num_cpus", "opensrv-mysql", @@ -4754,10 +4538,21 @@ dependencies = [ ] [[package]] -name = "sha2" +name = "sha1" version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9db03534dff993187064c4e0c05a5708d2a9728ace9a8959b77bedf415dac5" +checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" dependencies = [ "cfg-if", "cpufeatures", @@ -4991,6 +4786,7 @@ dependencies = [ "regex", "serde", "serde_json", + "sluice", "snafu", "store-api", "table", @@ -5136,9 +4932,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.99" +version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58dbef6ec655055e20b86b15a8cc6d439cca19b667537ac6a1369572d151ab13" +checksum = "52205623b1b0f064a4e71182c3b18ae902267282930c6d5462c91b859668426e" dependencies = [ "proc-macro2", "quote", @@ -5290,9 +5086,9 @@ dependencies = [ [[package]] name = "textwrap" -version = "0.15.0" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb" +checksum = "949517c0cf1bf4ee812e2e07e08ab448e3ae0d23472aee8a06c985f0c8815b16" [[package]] name = "thiserror" @@ -5437,7 +5233,7 @@ dependencies = [ "mio", "num_cpus", "once_cell", - "parking_lot 0.12.1", + "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", @@ -5467,16 +5263,6 @@ dependencies = [ "syn", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" -dependencies = [ - "native-tls", - "tokio", -] - [[package]] name = "tokio-postgres" version = "0.7.7" @@ -5490,7 +5276,7 @@ dependencies = [ "futures-channel", "futures-util", "log", - "parking_lot 0.12.1", + "parking_lot", "percent-encoding", "phf 0.11.1", "pin-project-lite", @@ -5502,10 +5288,21 @@ dependencies = [ ] [[package]] -name = "tokio-stream" -version = "0.1.9" +name = "tokio-rustls" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df54d54117d6fdc4e4fea40fe1e4e566b3505700e148a6827e59b34b0d2600d9" +checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" +dependencies = [ + "rustls", + "tokio", + "webpki", +] + +[[package]] +name = "tokio-stream" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6edf2d6bc038a43d31353570e27270603f4648d18f5ed10c0e179abe43255af" dependencies = [ "futures-core", "pin-project-lite", @@ -5919,9 +5716,9 @@ checksum = "dcc811dc4066ac62f84f11307873c4850cb653bfa9b1719cee2bd2204a4bc5dd" [[package]] name = "unicode-normalization" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854cbdc4f7bc6ae19c820d44abdc3277ac3e1b2b93db20a636825d9322fb60e6" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" dependencies = [ "tinyvec", ] @@ -5950,6 +5747,24 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "ureq" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97acb4c28a254fd7a4aeec976c46a7fa404eac4d7c134b30c75144846d7cb8f" +dependencies = [ + "base64", + "chunked_transfer", + "flate2", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "url", + "webpki", + "webpki-roots", +] + [[package]] name = "url" version = "2.3.1" @@ -6006,12 +5821,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8e76fae08f03f96e166d2dfda232190638c10e0383841252416f9cfe2ae60e6" -[[package]] -name = "waker-fn" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" - [[package]] name = "walkdir" version = "2.3.2" @@ -6033,12 +5842,6 @@ dependencies = [ "try-lock", ] -[[package]] -name = "wasi" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b89c3ce4ce14bdc6fb6beaf9ec7928ca331de5df7e5ea278375642a2f478570d" - [[package]] name = "wasi" version = "0.10.2+wasi-snapshot-preview1" @@ -6128,12 +5931,22 @@ dependencies = [ ] [[package]] -name = "wepoll-ffi" -version = "0.1.2" +name = "webpki" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d743fdedc5c64377b5fc2bc036b01c7fd642205a0d96356034ae3404d49eb7fb" +checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" dependencies = [ - "cc", + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1c760f0d366a6c24a02ed7816e23e691f5d92291f94d15e836006fd11b04daf" +dependencies = [ + "webpki", ] [[package]] diff --git a/README.md b/README.md index dead8eac22..54aa3476f4 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,6 @@ GreptimeDB: the next-generation hybrid timeseries/analytics processing database To compile GreptimeDB from source, you'll need the following: - Rust - Protobuf -- OpenSSL #### Rust @@ -23,23 +22,6 @@ The easiest way to install Rust is to use [`rustup`](https://rustup.rs/), which major package manager on macos and linux distributions. You can find an installation instructions [here](https://grpc.io/docs/protoc-installation/). -#### OpenSSL - -For Ubuntu: -```bash -sudo apt install libssl-dev -``` - -For RedHat-based: Fedora, Oracle Linux, etc: -```bash -sudo dnf install openssl-devel -``` - -For macOS: -```bash -brew install openssl -``` - ### Build the Docker Image ``` diff --git a/src/datanode/Cargo.toml b/src/datanode/Cargo.toml index 7bad02b547..88f469b572 100644 --- a/src/datanode/Cargo.toml +++ b/src/datanode/Cargo.toml @@ -54,7 +54,7 @@ version = "0.10" features = ["io_csv", "io_json", "io_parquet", "io_parquet_compression", "io_ipc", "ahash", "compute", "serde_types"] [dev-dependencies] -axum-test-helper = "0.1" +axum-test-helper = { git = "https://github.com/sunng87/axum-test-helper.git", branch = "patch-1" } client = { path = "../client" } common-query = { path = "../common/query" } datafusion = { git = "https://github.com/apache/arrow-datafusion.git", branch = "arrow2", features = ["simd"] } diff --git a/src/datanode/src/instance.rs b/src/datanode/src/instance.rs index 412a08a5b4..dc7b27ed76 100644 --- a/src/datanode/src/instance.rs +++ b/src/datanode/src/instance.rs @@ -12,7 +12,7 @@ use common_query::Output; use common_telemetry::logging::{debug, error, info}; use common_telemetry::timer; use log_store::fs::{config::LogConfig, log::LocalFileLogStore}; -use object_store::{backend::fs::Backend, util, ObjectStore}; +use object_store::{services::fs::Builder, util, ObjectStore}; use query::query_engine::{QueryEngineFactory, QueryEngineRef}; use servers::query_handler::{GrpcAdminHandler, GrpcQueryHandler, SqlQueryHandler}; use snafu::prelude::*; @@ -340,10 +340,9 @@ async fn new_object_store(store_config: &ObjectStoreConfig) -> Result Result<()> { async fn test_fs_backend() -> Result<()> { let tmp_dir = TempDir::new("test_fs_backend")?; let store = ObjectStore::new( - fs::Backend::build() + fs::Builder::default() .root(&tmp_dir.path().to_string_lossy()) - .finish() - .await?, + .build()?, ); test_object_crud(&store).await?; @@ -95,14 +94,14 @@ async fn test_s3_backend() -> Result<()> { logging::init_default_ut_logging(); if env::var("GT_S3_BUCKET").is_ok() { logging::info!("Running s3 test."); - let store = ObjectStore::new( - s3::Backend::build() - .access_key_id(&env::var("GT_S3_ACCESS_KEY_ID")?) - .secret_access_key(&env::var("GT_S3_ACCESS_KEY")?) - .bucket(&env::var("GT_S3_BUCKET")?) - .finish() - .await?, - ); + + let accessor = s3::Builder::default() + .access_key_id(&env::var("GT_S3_ACCESS_KEY_ID")?) + .secret_access_key(&env::var("GT_S3_ACCESS_KEY")?) + .bucket(&env::var("GT_S3_BUCKET")?) + .build()?; + + let store = ObjectStore::new(accessor); test_object_crud(&store).await?; test_object_list(&store).await?; } diff --git a/src/servers/Cargo.toml b/src/servers/Cargo.toml index f287ae28c3..ef6e47a9e4 100644 --- a/src/servers/Cargo.toml +++ b/src/servers/Cargo.toml @@ -39,10 +39,10 @@ tower = { version = "0.4", features = ["full"] } tower-http = { version = "0.3", features = ["full"] } [dev-dependencies] -axum-test-helper = "0.1" +axum-test-helper = { git = "https://github.com/sunng87/axum-test-helper.git", branch = "patch-1" } catalog = { path = "../catalog" } common-base = { path = "../common/base" } -mysql_async = "0.30" +mysql_async = { git = "https://github.com/Morranto/mysql_async.git", rev = "127b538" } rand = "0.8" script = { path = "../script", features = ["python"] } test-util = { path = "../../test-util" } diff --git a/src/storage/Cargo.toml b/src/storage/Cargo.toml index 0fda93c141..2826b1be90 100644 --- a/src/storage/Cargo.toml +++ b/src/storage/Cargo.toml @@ -28,6 +28,7 @@ prost = "0.11" regex = "1.5" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" +sluice = "0.5" snafu = { version = "0.7", features = ["backtraces"] } store-api = { path = "../store-api" } table = { path = "../table" } diff --git a/src/storage/src/engine.rs b/src/storage/src/engine.rs index 39dd931319..b6e704c8a2 100644 --- a/src/storage/src/engine.rs +++ b/src/storage/src/engine.rs @@ -314,7 +314,7 @@ impl EngineInner { mod tests { use datatypes::type_id::LogicalTypeId; use log_store::test_util::log_store_util; - use object_store::backend::fs::Backend; + use object_store::backend::fs::Builder; use store_api::storage::Region; use tempdir::TempDir; @@ -327,7 +327,8 @@ mod tests { log_store_util::create_tmp_local_file_log_store("test_engine_wal").await; let dir = TempDir::new("test_create_new_region").unwrap(); let store_dir = dir.path().to_string_lossy(); - let accessor = Backend::build().root(&store_dir).finish().await.unwrap(); + + let accessor = Builder::default().root(&store_dir).build().unwrap(); let object_store = ObjectStore::new(accessor); let config = EngineConfig::default(); diff --git a/src/storage/src/manifest/region.rs b/src/storage/src/manifest/region.rs index 9496285caf..df04aaf404 100644 --- a/src/storage/src/manifest/region.rs +++ b/src/storage/src/manifest/region.rs @@ -22,10 +22,9 @@ mod tests { common_telemetry::init_default_ut_logging(); let tmp_dir = TempDir::new("test_region_manifest").unwrap(); let object_store = ObjectStore::new( - fs::Backend::build() + fs::Builder::default() .root(&tmp_dir.path().to_string_lossy()) - .finish() - .await + .build() .unwrap(), ); diff --git a/src/storage/src/manifest/storage.rs b/src/storage/src/manifest/storage.rs index 4603c11302..c651d1c947 100644 --- a/src/storage/src/manifest/storage.rs +++ b/src/storage/src/manifest/storage.rs @@ -214,9 +214,12 @@ impl ManifestLogStorage for ManifestObjectStore { ); let bs = checkpoint_metadata.encode()?; - last_checkpoint.write(bs).await.context(WriteObjectSnafu { - path: last_checkpoint.path(), - })?; + last_checkpoint + .write(bs.as_ref()) + .await + .context(WriteObjectSnafu { + path: last_checkpoint.path(), + })?; Ok(()) } @@ -271,10 +274,9 @@ mod tests { common_telemetry::init_default_ut_logging(); let tmp_dir = TempDir::new("test_manifest_log_store").unwrap(); let object_store = ObjectStore::new( - fs::Backend::build() + fs::Builder::default() .root(&tmp_dir.path().to_string_lossy()) - .finish() - .await + .build() .unwrap(), ); diff --git a/src/storage/src/region/tests.rs b/src/storage/src/region/tests.rs index 9fb476262d..48c4e4ea05 100644 --- a/src/storage/src/region/tests.rs +++ b/src/storage/src/region/tests.rs @@ -183,10 +183,9 @@ async fn test_recover_region_manifets() { let tmp_dir = TempDir::new("test_new_region").unwrap(); let object_store = ObjectStore::new( - fs::Backend::build() + fs::Builder::default() .root(&tmp_dir.path().to_string_lossy()) - .finish() - .await + .build() .unwrap(), ); diff --git a/src/storage/src/sst/parquet.rs b/src/storage/src/sst/parquet.rs index 5d5c1c9ef3..d48c760b62 100644 --- a/src/storage/src/sst/parquet.rs +++ b/src/storage/src/sst/parquet.rs @@ -18,8 +18,9 @@ use datatypes::arrow::io::parquet::write::{ }; use futures::AsyncWriteExt; use futures_util::sink::SinkExt; -use futures_util::{Stream, TryStreamExt}; +use futures_util::{try_join, Stream, TryStreamExt}; use object_store::{ObjectStore, SeekableReader}; +use sluice::pipe; use snafu::ResultExt; use table::predicate::Predicate; @@ -62,45 +63,55 @@ impl<'a> ParquetWriter<'a> { let schema = store_schema.arrow_schema(); let object = self.object_store.object(self.file_path); - // FIXME(hl): writer size is not used in fs backend so just leave it to 0, - // but in s3/azblob backend the Content-Length field of HTTP request is set - // to this value. - let mut writer = object.writer(0).await.context(error::FlushIoSnafu)?; + let (reader, mut writer) = pipe::pipe(); // now all physical types use plain encoding, maybe let caller to choose encoding for each type. let encodings = get_encoding_for_schema(schema, |_| Encoding::Plain); - - let mut sink = FileSink::try_new( - &mut writer, - // The file sink needs the `Schema` instead of a reference. - (**schema).clone(), - encodings, - WriteOptions { - write_statistics: true, - compression: Compression::Gzip, - version: Version::V2, + try_join!( + async { + // FIXME(hl): writer size is not used in fs backend so just leave it to 0, + // but in s3/azblob backend the Content-Length field of HTTP request is set + // to this value. + object + .write_from(0, reader) + .await + .context(error::FlushIoSnafu) }, - ) - .context(error::WriteParquetSnafu)?; - - for batch in self.iter { - let batch = batch?; - sink.send(store_schema.batch_to_arrow_chunk(&batch)) - .await + async { + let mut sink = FileSink::try_new( + &mut writer, + // The file sink needs the `Schema` instead of a reference. + (**schema).clone(), + encodings, + WriteOptions { + write_statistics: true, + compression: Compression::Gzip, + version: Version::V2, + }, + ) .context(error::WriteParquetSnafu)?; - } - if let Some(meta) = extra_meta { - for (k, v) in meta { - sink.metadata.insert(k, Some(v)); + for batch in self.iter { + let batch = batch?; + sink.send(store_schema.batch_to_arrow_chunk(&batch)) + .await + .context(error::WriteParquetSnafu)?; + } + + if let Some(meta) = extra_meta { + for (k, v) in meta { + sink.metadata.insert(k, Some(v)); + } + } + sink.close().await.context(error::WriteParquetSnafu)?; + drop(sink); + + writer.close().await.context(error::WriteObjectSnafu { + path: self.file_path, + }) } - } - sink.close().await.context(error::WriteParquetSnafu)?; - - drop(sink); - writer.flush().await.context(error::WriteObjectSnafu { - path: self.file_path, - }) + ) + .map(|_| ()) } } @@ -287,7 +298,7 @@ mod tests { use datatypes::arrow::io::parquet::read::FileReader; use datatypes::prelude::{ScalarVector, Vector}; use datatypes::vectors::TimestampVector; - use object_store::backend::fs::Backend; + use object_store::backend::fs::Builder; use store_api::storage::OpType; use tempdir::TempDir; @@ -324,7 +335,7 @@ mod tests { let dir = TempDir::new("write_parquet").unwrap(); let path = dir.path().to_str().unwrap(); - let backend = Backend::build().root(path).finish().await.unwrap(); + let backend = Builder::default().root(path).build().unwrap(); let object_store = ObjectStore::new(backend); let sst_file_name = "test-flush.parquet"; let iter = memtable.iter(&IterContext::default()).unwrap(); diff --git a/src/storage/src/test_util/config_util.rs b/src/storage/src/test_util/config_util.rs index 46fbfc004e..a8fe7dfc24 100644 --- a/src/storage/src/test_util/config_util.rs +++ b/src/storage/src/test_util/config_util.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use log_store::fs::{config::LogConfig, log::LocalFileLogStore}; -use object_store::{backend::fs::Backend, ObjectStore}; +use object_store::{backend::fs::Builder, ObjectStore}; use crate::background::JobPoolImpl; use crate::engine; @@ -24,7 +24,7 @@ pub async fn new_store_config( let sst_dir = engine::region_sst_dir(parent_dir, region_name); let manifest_dir = engine::region_manifest_dir(parent_dir, region_name); - let accessor = Backend::build().root(store_dir).finish().await.unwrap(); + let accessor = Builder::default().root(store_dir).build().unwrap(); let object_store = ObjectStore::new(accessor); let sst_layer = Arc::new(FsAccessLayer::new(&sst_dir, object_store.clone())); let manifest = RegionManifest::new(&manifest_dir, object_store); diff --git a/src/table-engine/src/table/test_util.rs b/src/table-engine/src/table/test_util.rs index 82d8cb724f..aad856140d 100644 --- a/src/table-engine/src/table/test_util.rs +++ b/src/table-engine/src/table/test_util.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use datatypes::prelude::ConcreteDataType; use datatypes::schema::{ColumnSchema, Schema, SchemaBuilder, SchemaRef}; use log_store::fs::noop::NoopLogStore; -use object_store::{backend::fs::Backend, ObjectStore}; +use object_store::{services::fs::Builder, ObjectStore}; use storage::config::EngineConfig as StorageEngineConfig; use storage::EngineImpl; use table::engine::EngineContext; @@ -65,8 +65,7 @@ pub fn build_test_table_info() -> TableInfo { pub async fn new_test_object_store(prefix: &str) -> (TempDir, ObjectStore) { let dir = TempDir::new(prefix).unwrap(); let store_dir = dir.path().to_string_lossy(); - let accessor = Backend::build().root(&store_dir).finish().await.unwrap(); - + let accessor = Builder::default().root(&store_dir).build().unwrap(); (dir, ObjectStore::new(accessor)) }