From ebb51f16e0609d5929ef474c2b473c99e96cafd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s?= Date: Wed, 30 Nov 2022 16:47:32 +0100 Subject: [PATCH] Re-introduce aws-sdk-rust as rusoto S3 replacement (#2841) Part of https://github.com/neondatabase/neon/issues/2683 Initial PR: https://github.com/neondatabase/neon/pull/2802, revert: https://github.com/neondatabase/neon/pull/2837 Co-authored-by: andres --- Cargo.lock | 676 +++++++++++++++------------ libs/remote_storage/Cargo.toml | 7 +- libs/remote_storage/src/s3_bucket.rs | 197 ++++---- workspace_hack/Cargo.toml | 4 +- 4 files changed, 495 insertions(+), 389 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ac290dc59b..c9acb882eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -169,6 +169,333 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "aws-config" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56a636c44c77fa18bdba56126a34d30cfe5538fe88f7d34988fa731fee143ddd" +dependencies = [ + "aws-http", + "aws-sdk-sso", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-types", + "aws-types", + "bytes", + "hex", + "http", + "hyper", + "ring", + "time 0.3.15", + "tokio", + "tower", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-endpoint" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ca8f374874f6459aaa88dc861d7f5d834ca1ff97668eae190e97266b5f6c3fb" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "aws-types", + "http", + "regex", + "tracing", +] + +[[package]] +name = "aws-http" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78d41e19e779b73463f5f0c21b3aacc995f4ba783ab13a7ae9f5dfb159a551b4" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "aws-types", + "bytes", + "http", + "http-body", + "lazy_static", + "percent-encoding", + "pin-project-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-s3" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9f08665c8e03aca8cb092ef01e617436ebfa977fddc1240e1b062488ab5d48a" +dependencies = [ + "aws-endpoint", + "aws-http", + "aws-sig-auth", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-client", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "bytes-utils", + "http", + "http-body", + "tokio-stream", + "tower", + "tracing", +] + +[[package]] +name = "aws-sdk-sso" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86dcb1cb71aa8763b327542ead410424515cff0cde5b753eedd2917e09c63734" +dependencies = [ + "aws-endpoint", + "aws-http", + "aws-sig-auth", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-types", + "aws-types", + "bytes", + "http", + "tokio-stream", + "tower", +] + +[[package]] +name = "aws-sdk-sts" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdfcf584297c666f6b472d5368a78de3bc714b6e0a53d7fbf76c3e347c292ab1" +dependencies = [ + "aws-endpoint", + "aws-http", + "aws-sig-auth", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-query", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "http", + "tower", +] + +[[package]] +name = "aws-sig-auth" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12cbe7b2be9e185c1fbce27fc9c41c66b195b32d89aa099f98768d9544221308" +dependencies = [ + "aws-sigv4", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-types", + "http", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03ff4cff8c4a101962d593ba94e72cd83891aecd423f0c6e3146bff6fb92c9e3" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-http", + "bytes", + "form_urlencoded", + "hex", + "http", + "once_cell", + "percent-encoding", + "regex", + "ring", + "time 0.3.15", + "tracing", +] + +[[package]] +name = "aws-smithy-async" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b3442b4c5d3fc39891a2e5e625735fba6b24694887d49c6518460fde98247a9" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", + "tokio-stream", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc227e36e346f45298288359f37123e1a92628d1cec6b11b5eb335553278bd9e" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc32c", + "crc32fast", + "hex", + "http", + "http-body", + "md-5", + "pin-project-lite", + "sha1", + "sha2", + "tracing", +] + +[[package]] +name = "aws-smithy-client" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff28d553714f8f54cd921227934fc13a536a1c03f106e56b362fd57e16d450ad" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-types", + "bytes", + "fastrand", + "http", + "http-body", + "hyper", + "hyper-rustls", + "lazy_static", + "pin-project-lite", + "tokio", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7ea0df7161ce65b5c8ca6eb709a1a907376fa18226976e41c748ce02ccccf24" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf58ed4fefa61dbf038e5421a521cbc2c448ef69deff0ab1d915d8a10eda5664" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "http", + "http-body", + "hyper", + "once_cell", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "aws-smithy-http-tower" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20c96d7bd35e7cf96aca1134b2f81b1b59ffe493f7c6539c051791cbbf7a42d3" +dependencies = [ + "aws-smithy-http", + "bytes", + "http", + "http-body", + "pin-project-lite", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8324ba98c8a94187723cc16c37aefa09504646ee65c3d2c3af495bab5ea701b" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-query" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83834ed2ff69ea6f6657baf205267dc2c0abe940703503a3e5d60ce23be3d306" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-types" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b02e06ea63498c43bc0217ea4d16605d4e58d85c12fc23f6572ff6d0a840c61" +dependencies = [ + "itoa", + "num-integer", + "ryu", + "time 0.3.15", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "246e9f83dd1fdf5d347fa30ae4ad30a9d1d42ce4cd74a93d94afa874646f94cd" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05701d32da168b44f7ee63147781aed8723e792cc131cb9b18363b5393f17f70" +dependencies = [ + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-types", + "http", + "rustc_version 0.4.0", + "tracing", + "zeroize", +] + [[package]] name = "axum" version = "0.5.16" @@ -299,15 +626,6 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" -[[package]] -name = "block-buffer" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" -dependencies = [ - "generic-array", -] - [[package]] name = "block-buffer" version = "0.10.3" @@ -356,6 +674,16 @@ dependencies = [ "serde", ] +[[package]] +name = "bytes-utils" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e47d3a8076e283f3acd27400535992edb3ba4b5bb72f8891ad8fbe7932a7d4b9" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "cast" version = "0.3.0" @@ -806,16 +1134,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "crypto-mac" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714" -dependencies = [ - "generic-array", - "subtle", -] - [[package]] name = "cxx" version = "1.0.79" @@ -924,47 +1242,17 @@ dependencies = [ "rusticata-macros", ] -[[package]] -name = "digest" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" -dependencies = [ - "generic-array", -] - [[package]] name = "digest" version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adfbc57365a37acbd2ebf2b64d7e69bb766e2fea813521ed536f5d0520dcf86c" dependencies = [ - "block-buffer 0.10.3", + "block-buffer", "crypto-common", "subtle", ] -[[package]] -name = "dirs-next" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" -dependencies = [ - "cfg-if", - "dirs-sys-next", -] - -[[package]] -name = "dirs-sys-next" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" -dependencies = [ - "libc", - "redox_users", - "winapi", -] - [[package]] name = "displaydoc" version = "0.2.3" @@ -1109,21 +1397,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.1.0" @@ -1388,23 +1661,13 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ebdb29d2ea9ed0083cd8cece49bbd968021bd99b0849edb4a9a7ee0fdf6a4e0" -[[package]] -name = "hmac" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b" -dependencies = [ - "crypto-mac", - "digest 0.9.0", -] - [[package]] name = "hmac" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest 0.10.5", + "digest", ] [[package]] @@ -1495,7 +1758,9 @@ checksum = "d87c48c02e0dc5e3b849a2041db3029fd066650f8f717c07bf8ed78ccb895cac" dependencies = [ "http", "hyper", + "log", "rustls", + "rustls-native-certs", "tokio", "tokio-rustls", ] @@ -1512,19 +1777,6 @@ dependencies = [ "tokio-io-timeout", ] -[[package]] -name = "hyper-tls" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" -dependencies = [ - "bytes", - "hyper", - "native-tls", - "tokio", - "tokio-native-tls", -] - [[package]] name = "iana-time-zone" version = "0.1.51" @@ -1765,24 +2017,13 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb" -[[package]] -name = "md-5" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15" -dependencies = [ - "block-buffer 0.9.0", - "digest 0.9.0", - "opaque-debug", -] - [[package]] name = "md-5" version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" dependencies = [ - "digest 0.10.5", + "digest", ] [[package]] @@ -1873,24 +2114,6 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" -[[package]] -name = "native-tls" -version = "0.2.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9" -dependencies = [ - "lazy_static", - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - [[package]] name = "nb" version = "0.1.3" @@ -2061,57 +2284,12 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" -[[package]] -name = "opaque-debug" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" - -[[package]] -name = "openssl" -version = "0.10.42" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12fc0523e3bd51a692c8850d075d74dc062ccf251c0110668cbd921917118a13" -dependencies = [ - "bitflags", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "openssl-probe" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" -[[package]] -name = "openssl-sys" -version = "0.9.76" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5230151e44c0f05157effb743e8d517472843121cf9243e8b81393edb5acd9ce" -dependencies = [ - "autocfg", - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "os_str_bytes" version = "6.3.0" @@ -2333,12 +2511,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "pkg-config" -version = "0.3.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" - [[package]] name = "plotters" version = "0.3.4" @@ -2389,12 +2561,12 @@ dependencies = [ "byteorder", "bytes", "fallible-iterator", - "hmac 0.12.1", + "hmac", "lazy_static", - "md-5 0.10.5", + "md-5", "memchr", "rand", - "sha2 0.10.6", + "sha2", "stringprep", ] @@ -2688,7 +2860,7 @@ dependencies = [ "git-version", "hashbrown", "hex", - "hmac 0.12.1", + "hmac", "hyper", "itertools", "md5", @@ -2707,7 +2879,7 @@ dependencies = [ "scopeguard", "serde", "serde_json", - "sha2 0.10.6", + "sha2", "socket2", "thiserror", "tokio", @@ -2826,17 +2998,6 @@ dependencies = [ "bitflags", ] -[[package]] -name = "redox_users" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" -dependencies = [ - "getrandom", - "redox_syscall", - "thiserror", -] - [[package]] name = "regex" version = "1.6.0" @@ -2869,10 +3030,13 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", + "aws-config", + "aws-sdk-s3", + "aws-smithy-http", + "aws-types", + "hyper", "metrics", "once_cell", - "rusoto_core", - "rusoto_s3", "serde", "serde_json", "tempfile", @@ -3026,88 +3190,6 @@ dependencies = [ "syn", ] -[[package]] -name = "rusoto_core" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1db30db44ea73551326269adcf7a2169428a054f14faf9e1768f2163494f2fa2" -dependencies = [ - "async-trait", - "base64", - "bytes", - "crc32fast", - "futures", - "http", - "hyper", - "hyper-tls", - "lazy_static", - "log", - "rusoto_credential", - "rusoto_signature", - "rustc_version 0.4.0", - "serde", - "serde_json", - "tokio", - "xml-rs", -] - -[[package]] -name = "rusoto_credential" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee0a6c13db5aad6047b6a44ef023dbbc21a056b6dab5be3b79ce4283d5c02d05" -dependencies = [ - "async-trait", - "chrono", - "dirs-next", - "futures", - "hyper", - "serde", - "serde_json", - "shlex", - "tokio", - "zeroize", -] - -[[package]] -name = "rusoto_s3" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7aae4677183411f6b0b412d66194ef5403293917d66e70ab118f07cc24c5b14d" -dependencies = [ - "async-trait", - "bytes", - "futures", - "rusoto_core", - "xml-rs", -] - -[[package]] -name = "rusoto_signature" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5ae95491c8b4847931e291b151127eccd6ff8ca13f33603eb3d0035ecb05272" -dependencies = [ - "base64", - "bytes", - "chrono", - "digest 0.9.0", - "futures", - "hex", - "hmac 0.11.0", - "http", - "hyper", - "log", - "md-5 0.9.1", - "percent-encoding", - "pin-project-lite", - "rusoto_credential", - "rustc_version 0.4.0", - "serde", - "sha2 0.9.9", - "tokio", -] - [[package]] name = "rustc-demangle" version = "0.1.21" @@ -3159,6 +3241,18 @@ dependencies = [ "webpki", ] +[[package]] +name = "rustls-native-certs" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pemfile" version = "1.0.1" @@ -3400,16 +3494,14 @@ dependencies = [ ] [[package]] -name = "sha2" -version = "0.9.9" +name = "sha1" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" +checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" dependencies = [ - "block-buffer 0.9.0", "cfg-if", "cpufeatures", - "digest 0.9.0", - "opaque-debug", + "digest", ] [[package]] @@ -3420,7 +3512,7 @@ checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" dependencies = [ "cfg-if", "cpufeatures", - "digest 0.10.5", + "digest", ] [[package]] @@ -3835,16 +3927,6 @@ dependencies = [ "syn", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" -dependencies = [ - "native-tls", - "tokio", -] - [[package]] name = "tokio-postgres" version = "0.7.6" @@ -4227,6 +4309,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "urlencoding" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9" + [[package]] name = "utils" version = "0.1.0" @@ -4294,12 +4382,6 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77439c1b53d2303b20d9459b1ade71a83c716e3f9c34f3228c00e6f185d6c002" -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - [[package]] name = "version_check" version = "0.9.4" @@ -4565,7 +4647,6 @@ dependencies = [ "ahash", "anyhow", "bytes", - "chrono", "clap 4.0.15", "crossbeam-utils", "either", @@ -4595,6 +4676,7 @@ dependencies = [ "time 0.3.15", "tokio", "tokio-util", + "tower", "tracing", "tracing-core", ] @@ -4627,10 +4709,10 @@ dependencies = [ ] [[package]] -name = "xml-rs" -version = "0.8.4" +name = "xmlparser" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2d7d3948613f75c98fd9328cfdcc45acc4d360655289d0a7d4ec931392200a3" +checksum = "4d25c75bf9ea12c4040a97f829154768bbbce366287e2dc044af160cd79a13fd" [[package]] name = "yasna" diff --git a/libs/remote_storage/Cargo.toml b/libs/remote_storage/Cargo.toml index f54d91905c..ebd30fc1eb 100644 --- a/libs/remote_storage/Cargo.toml +++ b/libs/remote_storage/Cargo.toml @@ -9,8 +9,11 @@ async-trait = "0.1" metrics = { version = "0.1", path = "../metrics" } utils = { version = "0.1", path = "../utils" } once_cell = "1.13.0" -rusoto_core = "0.48" -rusoto_s3 = "0.48" +aws-smithy-http = "0.51.0" +aws-types = "0.51.0" +aws-config = { version = "0.51.0", default-features = false, features=["rustls"] } +aws-sdk-s3 = "0.21.0" +hyper = { version = "0.14", features = ["stream"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1" tokio = { version = "1.17", features = ["sync", "macros", "fs", "io-util"] } diff --git a/libs/remote_storage/src/s3_bucket.rs b/libs/remote_storage/src/s3_bucket.rs index 74632430cd..c721560c29 100644 --- a/libs/remote_storage/src/s3_bucket.rs +++ b/libs/remote_storage/src/s3_bucket.rs @@ -4,27 +4,36 @@ //! allowing multiple api users to independently work with the same S3 bucket, if //! their bucket prefixes are both specified and different. +use std::env::var; use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; use anyhow::Context; -use rusoto_core::{ - credential::{InstanceMetadataProvider, StaticProvider}, - HttpClient, Region, RusotoError, +use aws_config::{ + environment::credentials::EnvironmentVariableCredentialsProvider, imds, + imds::credentials::ImdsCredentialsProvider, meta::credentials::provide_credentials_fn, }; -use rusoto_s3::{ - DeleteObjectRequest, GetObjectError, GetObjectRequest, ListObjectsV2Request, PutObjectRequest, - S3Client, StreamingBody, S3, +use aws_sdk_s3::{ + config::Config, + error::{GetObjectError, GetObjectErrorKind}, + types::{ByteStream, SdkError}, + Client, Endpoint, Region, }; +use aws_smithy_http::body::SdkBody; +use aws_types::credentials::{CredentialsError, ProvideCredentials}; +use hyper::Body; use tokio::{io, sync::Semaphore}; use tokio_util::io::ReaderStream; use tracing::debug; +use super::StorageMetadata; use crate::{ strip_path_prefix, Download, DownloadError, RemoteObjectId, RemoteStorage, S3Config, REMOTE_STORAGE_PREFIX_SEPARATOR, }; -use super::StorageMetadata; +const DEFAULT_IMDS_TIMEOUT: Duration = Duration::from_secs(10); pub(super) mod metrics { use metrics::{register_int_counter_vec, IntCounterVec}; @@ -116,7 +125,7 @@ fn download_destination( /// AWS S3 storage. pub struct S3Bucket { workdir: PathBuf, - client: S3Client, + client: Client, bucket_name: String, prefix_in_bucket: Option, // Every request to S3 can be throttled or cancelled, if a certain number of requests per second is exceeded. @@ -125,6 +134,12 @@ pub struct S3Bucket { concurrency_limiter: Semaphore, } +#[derive(Default)] +struct GetObjectRequest { + bucket: String, + key: String, + range: Option, +} impl S3Bucket { /// Creates the S3 storage, errors if incorrect AWS S3 configuration provided. pub fn new(aws_config: &S3Config, workdir: PathBuf) -> anyhow::Result { @@ -132,43 +147,40 @@ impl S3Bucket { "Creating s3 remote storage for S3 bucket {}", aws_config.bucket_name ); - let region = match aws_config.endpoint.clone() { - Some(custom_endpoint) => Region::Custom { - name: aws_config.bucket_region.clone(), - endpoint: custom_endpoint, - }, - None => aws_config - .bucket_region - .parse::() - .context("Failed to parse the s3 region from config")?, - }; - let request_dispatcher = HttpClient::new().context("Failed to create S3 http client")?; + let mut config_builder = Config::builder() + .region(Region::new(aws_config.bucket_region.clone())) + .credentials_provider(provide_credentials_fn(|| async { + match var("AWS_ACCESS_KEY_ID").is_ok() && var("AWS_SECRET_ACCESS_KEY").is_ok() { + true => { + EnvironmentVariableCredentialsProvider::new() + .provide_credentials() + .await + } + false => { + let imds_client = imds::Client::builder() + .connect_timeout(DEFAULT_IMDS_TIMEOUT) + .read_timeout(DEFAULT_IMDS_TIMEOUT) + .build() + .await + .map_err(CredentialsError::unhandled)?; + ImdsCredentialsProvider::builder() + .imds_client(imds_client) + .build() + .provide_credentials() + .await + } + } + })); - let access_key_id = std::env::var("AWS_ACCESS_KEY_ID").ok(); - let secret_access_key = std::env::var("AWS_SECRET_ACCESS_KEY").ok(); - // session token is used when authorizing through sso - // which is typically the case when testing locally on developer machine - let session_token = std::env::var("AWS_SESSION_TOKEN").ok(); - - let client = if access_key_id.is_none() && secret_access_key.is_none() { - debug!("Using IAM-based AWS access"); - S3Client::new_with(request_dispatcher, InstanceMetadataProvider::new(), region) - } else { - debug!( - "Using credentials-based AWS access. Session token is set: {}", - session_token.is_some() + if let Some(custom_endpoint) = aws_config.endpoint.clone() { + let endpoint = Endpoint::immutable( + custom_endpoint + .parse() + .expect("Failed to parse S3 custom endpoint"), ); - S3Client::new_with( - request_dispatcher, - StaticProvider::new( - access_key_id.unwrap_or_default(), - secret_access_key.unwrap_or_default(), - session_token, - None, - ), - region, - ) - }; + config_builder.set_endpoint_resolver(Some(Arc::new(endpoint))); + } + let client = Client::from_conf(config_builder.build()); let prefix_in_bucket = aws_config.prefix_in_bucket.as_deref().map(|prefix| { let mut prefix = prefix; @@ -182,7 +194,6 @@ impl S3Bucket { } prefix }); - Ok(Self { client, workdir, @@ -202,20 +213,33 @@ impl S3Bucket { metrics::inc_get_object(); - match self.client.get_object(request).await { - Ok(object_output) => match object_output.body { - None => { - metrics::inc_get_object_fail(); - Err(DownloadError::Other(anyhow::anyhow!( - "Got no body for the S3 object given" - ))) - } - Some(body) => Ok(Download { - metadata: object_output.metadata.map(StorageMetadata), - download_stream: Box::pin(io::BufReader::new(body.into_async_read())), - }), - }, - Err(RusotoError::Service(GetObjectError::NoSuchKey(_))) => Err(DownloadError::NotFound), + let get_object = self + .client + .get_object() + .bucket(request.bucket) + .key(request.key) + .set_range(request.range) + .send() + .await; + + match get_object { + Ok(object_output) => { + let metadata = object_output.metadata().cloned().map(StorageMetadata); + Ok(Download { + metadata, + download_stream: Box::pin(io::BufReader::new( + object_output.body.into_async_read(), + )), + }) + } + Err(SdkError::ServiceError { + err: + GetObjectError { + kind: GetObjectErrorKind::NoSuchKey(..), + .. + }, + .. + }) => Err(DownloadError::NotFound), Err(e) => { metrics::inc_get_object_fail(); Err(DownloadError::Other(anyhow::anyhow!( @@ -261,12 +285,11 @@ impl RemoteStorage for S3Bucket { let fetch_response = self .client - .list_objects_v2(ListObjectsV2Request { - bucket: self.bucket_name.clone(), - prefix: self.prefix_in_bucket.clone(), - continuation_token, - ..ListObjectsV2Request::default() - }) + .list_objects_v2() + .bucket(self.bucket_name.clone()) + .set_prefix(self.prefix_in_bucket.clone()) + .set_continuation_token(continuation_token) + .send() .await .map_err(|e| { metrics::inc_list_objects_fail(); @@ -322,13 +345,12 @@ impl RemoteStorage for S3Bucket { let fetch_response = self .client - .list_objects_v2(ListObjectsV2Request { - bucket: self.bucket_name.clone(), - prefix: list_prefix.clone(), - continuation_token, - delimiter: Some(REMOTE_STORAGE_PREFIX_SEPARATOR.to_string()), - ..ListObjectsV2Request::default() - }) + .list_objects_v2() + .bucket(self.bucket_name.clone()) + .set_prefix(list_prefix.clone()) + .set_continuation_token(continuation_token) + .delimiter(REMOTE_STORAGE_PREFIX_SEPARATOR.to_string()) + .send() .await .map_err(|e| { metrics::inc_list_objects_fail(); @@ -366,17 +388,18 @@ impl RemoteStorage for S3Bucket { .context("Concurrency limiter semaphore got closed during S3 upload")?; metrics::inc_put_object(); + + let body = Body::wrap_stream(ReaderStream::new(from)); + let bytes_stream = ByteStream::new(SdkBody::from(body)); + self.client - .put_object(PutObjectRequest { - body: Some(StreamingBody::new_with_size( - ReaderStream::new(from), - from_size_bytes, - )), - bucket: self.bucket_name.clone(), - key: to.0.to_owned(), - metadata: metadata.map(|m| m.0), - ..PutObjectRequest::default() - }) + .put_object() + .bucket(self.bucket_name.clone()) + .key(to.0.to_owned()) + .set_metadata(metadata.map(|m| m.0)) + .content_length(from_size_bytes.try_into()?) + .body(bytes_stream) + .send() .await .map_err(|e| { metrics::inc_put_object_fail(); @@ -412,7 +435,6 @@ impl RemoteStorage for S3Bucket { bucket: self.bucket_name.clone(), key: from.0.to_owned(), range, - ..GetObjectRequest::default() }) .await } @@ -427,11 +449,10 @@ impl RemoteStorage for S3Bucket { metrics::inc_delete_object(); self.client - .delete_object(DeleteObjectRequest { - bucket: self.bucket_name.clone(), - key: remote_object_id.0.to_owned(), - ..DeleteObjectRequest::default() - }) + .delete_object() + .bucket(self.bucket_name.clone()) + .key(remote_object_id.0.to_owned()) + .send() .await .map_err(|e| { metrics::inc_delete_object_fail(); @@ -600,7 +621,7 @@ mod tests { fn dummy_storage(workdir: PathBuf) -> S3Bucket { S3Bucket { workdir, - client: S3Client::new("us-east-1".parse().unwrap()), + client: Client::new(&aws_config::SdkConfig::builder().build()), bucket_name: "dummy-bucket".to_string(), prefix_in_bucket: Some("dummy_prefix/".to_string()), concurrency_limiter: Semaphore::new(1), diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index 00c590a0ca..31e4426ac2 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -16,7 +16,6 @@ publish = false ahash = { version = "0.7", features = ["std"] } anyhow = { version = "1", features = ["backtrace", "std"] } bytes = { version = "1", features = ["serde", "std"] } -chrono = { version = "0.4", features = ["clock", "iana-time-zone", "js-sys", "oldtime", "serde", "std", "time", "wasm-bindgen", "wasmbind", "winapi"] } clap = { version = "4", features = ["color", "derive", "error-context", "help", "std", "string", "suggestions", "usage"] } crossbeam-utils = { version = "0.8", features = ["once_cell", "std"] } either = { version = "1", features = ["use_std"] } @@ -31,7 +30,7 @@ log = { version = "0.4", default-features = false, features = ["serde", "std"] } memchr = { version = "2", features = ["std"] } nom = { version = "7", features = ["alloc", "std"] } num-bigint = { version = "0.4", features = ["std"] } -num-integer = { version = "0.1", default-features = false, features = ["i128", "std"] } +num-integer = { version = "0.1", features = ["i128", "std"] } num-traits = { version = "0.2", features = ["i128", "libm", "std"] } prost-93f6ce9d446188ac = { package = "prost", version = "0.10", features = ["prost-derive", "std"] } prost-a6292c17cd707f01 = { package = "prost", version = "0.11", features = ["prost-derive", "std"] } @@ -45,6 +44,7 @@ stable_deref_trait = { version = "1", features = ["alloc", "std"] } time = { version = "0.3", features = ["alloc", "formatting", "itoa", "macros", "parsing", "std", "time-macros"] } tokio = { version = "1", features = ["bytes", "fs", "io-std", "io-util", "libc", "macros", "memchr", "mio", "net", "num_cpus", "once_cell", "process", "rt", "rt-multi-thread", "signal-hook-registry", "socket2", "sync", "time", "tokio-macros"] } tokio-util = { version = "0.7", features = ["codec", "io", "io-util", "tracing"] } +tower = { version = "0.4", features = ["__common", "balance", "buffer", "discover", "futures-core", "futures-util", "indexmap", "limit", "load", "log", "make", "pin-project", "pin-project-lite", "rand", "ready-cache", "retry", "slab", "timeout", "tokio", "tokio-util", "tracing", "util"] } tracing = { version = "0.1", features = ["attributes", "log", "std", "tracing-attributes"] } tracing-core = { version = "0.1", features = ["once_cell", "std"] }