rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-10-03 10:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add HDFS object storage backend (#8701)

Browse Source 
* feat: add HDFS object storage backend

Signed-off-by: Minghan2005 <cambrianocean@gmail.com>

* fix: make HDFS storage operations durable

Gate the native HDFS backend behind an explicit feature. Publish writes through same-directory temporary files and atomic HDFS Rename2 replacement, and provide streaming copy fallback for COPY_REGION. Add regression coverage for interrupted writes and the region-copy path.

Signed-off-by: Minghan2005 <cambrianocean@gmail.com>

* ci: run HDFS object store tests

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

* docs: note HDFS temporary file cleanup follow-up

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

* feat: enable HDFS object storage by default

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

* docs: remove redundant HDFS build feature notes

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

---------

Signed-off-by: Minghan2005 <cambrianocean@gmail.com>
Signed-off-by: jeremyhi <fengjiachun@gmail.com>
Co-authored-by: Minghan2005 <cambrianocean@gmail.com>
Co-authored-by: jeremyhi <fengjiachun@gmail.com>
This commit is contained in:
Minghan Jiang
2026-09-28 08:39:48 +00:00
committed by GitHub
co-authored by Minghan2005 jeremyhi
parent ffdd6d09a6
commit 75bd8e9ce6
17 changed files with 1090 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+205 -36
View File
@@ -40,10 +40,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0"
dependencies = [
"cfg-if",
"cipher",
"cipher 0.4.4",
"cpufeatures 0.2.17",
]
[[package]]
name = "aes"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35f0f96ce78e38c3dc6d8948aa8163d06385be74000f3c7a95bf1eef35d3ea32"
dependencies = [
"cipher 0.5.2",
"cpubits",
"cpufeatures 0.3.0",
]
[[package]]
name = "aes-siv"
version = "0.7.0"
@@ -51,10 +62,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e08d0cdb774acd1e4dac11478b1a0c0d203134b2aab0ba25eb430de9b18f8b9"
dependencies = [
"aead",
"aes",
"cipher",
"aes 0.8.4",
"cipher 0.4.4",
"cmac",
"ctr",
"ctr 0.9.2",
"dbl",
"digest 0.10.7",
"zeroize",
@@ -1400,6 +1411,15 @@ dependencies = [
"generic-array",
]
[[package]]
name = "block-padding"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "710f1dd022ef4e93f8a438b4ba958de7f64308434fa6a87104481645cc30068b"
dependencies = [
"hybrid-array",
]
[[package]]
name = "blocking"
version = "1.6.1"
@@ -1534,9 +1554,9 @@ checksum = "40e38929add23cdf8a366df9b0e088953150724bcbe5fc330b0d8eb3b328eec8"
[[package]]
name = "bumpalo"
version = "3.19.0"
version = "3.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43"
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
[[package]]
name = "bytecheck"
@@ -1730,7 +1750,16 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6"
dependencies = [
"cipher",
"cipher 0.4.4",
]
[[package]]
name = "cbc"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce2dc9ee5f88d11e0beb842c88b33c8a5cf0d1329c4b19494af42b07dbfe8896"
dependencies = [
"cipher 0.5.2",
]
[[package]]
@@ -1775,7 +1804,7 @@ version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "738b8d467867f80a71351933f70461f5b56f24d5c93e0cf216e59229c968d330"
dependencies = [
"cipher",
"cipher 0.4.4",
]
[[package]]
@@ -1827,7 +1856,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818"
dependencies = [
"cfg-if",
"cipher",
"cipher 0.4.4",
"cpufeatures 0.2.17",
]
@@ -1850,7 +1879,7 @@ checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35"
dependencies = [
"aead",
"chacha20 0.9.1",
"cipher",
"cipher 0.4.4",
"poly1305",
"zeroize",
]
@@ -1943,10 +1972,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
dependencies = [
"crypto-common 0.1.6",
"inout",
"inout 0.1.4",
"zeroize",
]
[[package]]
name = "cipher"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c"
dependencies = [
"block-buffer 0.12.0",
"crypto-common 0.2.2",
"inout 0.2.2",
]
[[package]]
name = "clang-sys"
version = "1.8.1"
@@ -1955,7 +1995,7 @@ checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"
dependencies = [
"glob",
"libc",
"libloading",
"libloading 0.8.8",
]
[[package]]
@@ -2118,7 +2158,7 @@ version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8543454e3c3f5126effff9cd44d562af4e31fb8ce1cc0d3dcd8f084515dbc1aa"
dependencies = [
"cipher",
"cipher 0.4.4",
"dbl",
"digest 0.10.7",
]
@@ -2228,7 +2268,7 @@ checksum = "fe6d2e5af09e8c8ad56c969f2157a3d4238cebc7c55f0a517728c38f7b200f81"
dependencies = [
"serde",
"termcolor",
"unicode-width 0.1.14",
"unicode-width 0.2.1",
]
[[package]]
@@ -3218,6 +3258,12 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "cpubits"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae"
[[package]]
name = "cpufeatures"
version = "0.2.17"
@@ -3238,9 +3284,9 @@ dependencies = [
[[package]]
name = "crc"
version = "3.3.0"
version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675"
checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d"
dependencies = [
"crc-catalog",
]
@@ -3438,7 +3484,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9d6cf87adf719ddf43a805e92c6870a531aedda35ff640442cbaf8674e141e1"
dependencies = [
"aead",
"cipher",
"cipher 0.4.4",
"generic-array",
"poly1305",
"salsa20",
@@ -3483,7 +3529,16 @@ version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835"
dependencies = [
"cipher",
"cipher 0.4.4",
]
[[package]]
name = "ctr"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baaca1c4b237092596f64d571e9db6ce4109c4ef9742e27590f1709594461f21"
dependencies = [
"cipher 0.5.2",
]
[[package]]
@@ -4742,6 +4797,15 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "des"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "916a94e407b54f9034d71dd748234cd1e516ced6284009906ae246f177eafe5a"
dependencies = [
"cipher 0.5.2",
]
[[package]]
name = "diff"
version = "0.1.13"
@@ -5819,6 +5883,34 @@ dependencies = [
"byteorder",
]
[[package]]
name = "g2gen"
version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c5a7e0eb46f83a20260b850117d204366674e85d3a908d90865c78df9a6b1dfc"
dependencies = [
"g2poly",
"proc-macro2",
"quote",
"syn 2.0.117",
]
[[package]]
name = "g2p"
version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "539e2644c030d3bf4cd208cb842d2ce2f80e82e6e8472390bcef83ceba0d80ad"
dependencies = [
"g2gen",
"g2poly",
]
[[package]]
name = "g2poly"
version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "312d2295c7302019c395cfb90dacd00a82a2eabd700429bba9c7a3f38dbbe11b"
[[package]]
name = "generator"
version = "0.8.5"
@@ -6167,6 +6259,47 @@ dependencies = [
"hashbrown 0.15.4",
]
[[package]]
name = "hdfs-native"
version = "0.14.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fd181084003308224efddf737417832186839ce2882d2468ddad0114cfb3551"
dependencies = [
"aes 0.9.3",
"base64 0.22.1",
"bitflags 2.12.1",
"bumpalo",
"bytes",
"cbc 0.2.1",
"chrono",
"cipher 0.5.2",
"crc",
"ctr 0.10.1",
"des",
"dns-lookup",
"futures",
"g2p",
"hex",
"hmac 0.13.0",
"libc",
"libloading 0.9.0",
"log",
"md-5 0.11.0",
"num-traits",
"once_cell",
"prost 0.14.1",
"prost-types 0.14.1",
"rand 0.10.1",
"regex",
"roxmltree",
"socket2 0.6.4",
"thiserror 2.0.17",
"tokio",
"url",
"uuid",
"whoami 2.1.3",
]
[[package]]
name = "hdrhistogram"
version = "7.5.4"
@@ -6549,7 +6682,7 @@ dependencies = [
"libc",
"percent-encoding",
"pin-project-lite",
"socket2 0.5.10",
"socket2 0.6.4",
"tokio",
"tower-service",
"tracing",
@@ -6620,7 +6753,7 @@ dependencies = [
"js-sys",
"log",
"wasm-bindgen",
"windows-core 0.57.0",
"windows-core 0.61.2",
]
[[package]]
@@ -6996,10 +7129,20 @@ version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
dependencies = [
"block-padding",
"block-padding 0.3.3",
"generic-array",
]
[[package]]
name = "inout"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7"
dependencies = [
"block-padding 0.4.2",
"hybrid-array",
]
[[package]]
name = "instant"
version = "0.1.13"
@@ -7061,7 +7204,7 @@ version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96e4f67dbfc0f75d7b65953ecf0be3fd84ee0cb1ae72a00a4aa9a2f5518a2c80"
dependencies = [
"aes",
"aes 0.8.4",
]
[[package]]
@@ -7810,6 +7953,16 @@ dependencies = [
"windows-targets 0.53.5",
]
[[package]]
name = "libloading"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60"
dependencies = [
"cfg-if",
"windows-link 0.2.1",
]
[[package]]
name = "liblzma"
version = "0.4.6"
@@ -9300,7 +9453,7 @@ version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77e878c846a8abae00dd069496dbe8751b16ac1c3d6bd2a7283a938e8228f90d"
dependencies = [
"proc-macro-crate 1.3.1",
"proc-macro-crate 3.3.0",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -9407,6 +9560,7 @@ dependencies = [
"common-test-util",
"derive_builder 0.20.2",
"futures",
"hdfs-native",
"humantime-serde",
"object-store",
"object_store",
@@ -9482,7 +9636,7 @@ version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2cc40678e045ff4eb1666ea6c0f994b133c31f673c09aed292261b6d5b6963a0"
dependencies = [
"cipher",
"cipher 0.4.4",
]
[[package]]
@@ -9555,6 +9709,7 @@ dependencies = [
"opendal-service-azblob",
"opendal-service-fs",
"opendal-service-gcs",
"opendal-service-hdfs-native",
"opendal-service-http",
"opendal-service-mysql",
"opendal-service-oss",
@@ -9745,6 +9900,20 @@ dependencies = [
"uuid",
]
[[package]]
name = "opendal-service-hdfs-native"
version = "0.59.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b84340e11a3bcd7f1f0b55df032d040b2907483c546c3021485aba749c3072c"
dependencies = [
"bytes",
"futures",
"hdfs-native",
"log",
"opendal-core",
"serde",
]
[[package]]
name = "opendal-service-http"
version = "0.59.2"
@@ -10792,8 +10961,8 @@ version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6"
dependencies = [
"aes",
"cbc",
"aes 0.8.4",
"cbc 0.1.2",
"der",
"pbkdf2",
"scrypt",
@@ -11313,7 +11482,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4"
dependencies = [
"bytes",
"heck 0.4.1",
"heck 0.5.0",
"itertools 0.12.1",
"log",
"multimap",
@@ -11333,8 +11502,8 @@ version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac6c3320f9abac597dcbc668774ef006702672474aad53c6d596b62e487b40b1"
dependencies = [
"heck 0.4.1",
"itertools 0.10.5",
"heck 0.5.0",
"itertools 0.14.0",
"log",
"multimap",
"once_cell",
@@ -11382,7 +11551,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
dependencies = [
"anyhow",
"itertools 0.10.5",
"itertools 0.14.0",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -11395,7 +11564,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425"
dependencies = [
"anyhow",
"itertools 0.10.5",
"itertools 0.14.0",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -12988,7 +13157,7 @@ version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213"
dependencies = [
"cipher",
"cipher 0.4.4",
]
[[package]]
@@ -13778,7 +13947,7 @@ version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1961e2ef424c1424204d3a5d6975f934f56b6d50ff5732382d84ebf460e147f7"
dependencies = [
"heck 0.4.1",
"heck 0.5.0",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -16119,13 +16288,13 @@ version = "0.33.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "772206835b5bca8719825d2ab9e6fb160cbfce8ac1dc0e5dffe22cbe77254468"
dependencies = [
"aes",
"aes 0.8.4",
"aes-siv",
"base16",
"base62",
"base64-simd",
"bytes",
"cbc",
"cbc 0.1.2",
"cfb-mode",
"cfg-if",
"chacha20poly1305",
@@ -16141,7 +16310,7 @@ dependencies = [
"crc",
"crypto_secretbox",
"csv",
"ctr",
"ctr 0.9.2",
"digest 0.10.7",
"dns-lookup",
"domain",