rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-10-15 00:12:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add HDFS object storage backend (#8701)

Browse Source 
* feat: add HDFS object storage backend

Signed-off-by: Minghan2005 <cambrianocean@gmail.com>

* fix: make HDFS storage operations durable

Gate the native HDFS backend behind an explicit feature. Publish writes through same-directory temporary files and atomic HDFS Rename2 replacement, and provide streaming copy fallback for COPY_REGION. Add regression coverage for interrupted writes and the region-copy path.

Signed-off-by: Minghan2005 <cambrianocean@gmail.com>

* ci: run HDFS object store tests

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

* docs: note HDFS temporary file cleanup follow-up

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

* feat: enable HDFS object storage by default

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

* docs: remove redundant HDFS build feature notes

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

---------

Signed-off-by: Minghan2005 <cambrianocean@gmail.com>
Signed-off-by: jeremyhi <fengjiachun@gmail.com>
Co-authored-by: Minghan2005 <cambrianocean@gmail.com>
Co-authored-by: jeremyhi <fengjiachun@gmail.com>
This commit is contained in:
Minghan Jiang
2026-09-28 08:39:48 +00:00
committed by GitHub
co-authored by Minghan2005 jeremyhi
parent ffdd6d09a6
commit 75bd8e9ce6
17 changed files with 1090 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/cmd/Cargo.toml
+2
View File
@@ -22,6 +22,7 @@ required-features = ["dev-tools"]
[features]
default = [
"ai_functions",
"hdfs-object-store",
"servers/pprof",
"servers/mem-prof",
"meta-srv/pg_kvbackend",
@@ -32,6 +33,7 @@ enterprise = ["common-meta/enterprise", "frontend/enterprise", "meta-srv/enterpr
# Developer-only helper binaries and diagnostic datanode commands.
# Kept out of `default` so normal/release builds don't compile them.
dev-tools = []
hdfs-object-store = ["mito2/hdfs-object-store", "object-store/hdfs-object-store"]
mysql-object-store = ["object-store/mysql-object-store"]
tokio-console = ["common-telemetry/tokio-console"]
src/cmd/tests/load_config_test.rs
+38
View File
@@ -125,6 +125,44 @@ fn test_load_runtime_options_without_max_blocking_threads() {
);
}
#[test]
#[cfg(feature = "hdfs-object-store")]
fn test_load_datanode_hdfs_config() {
let config = tempfile::NamedTempFile::new().unwrap();
std::fs::write(
config.path(),
r#"
[storage]
type = "Hdfs"
name = "local-hdfs"
root = "/greptimedb"
name_node = "hdfs://127.0.0.1:9000"
enable_read_cache = false
options = { "dfs.client.block.write.replace-datanode-on-failure.enable" = "true" }
"#,
)
.unwrap();
let options =
GreptimeOptions::<DatanodeOptions>::load_layered_options(config.path().to_str(), "")
.unwrap();
let object_store::config::ObjectStoreConfig::Hdfs(hdfs) = options.component.storage.store
else {
unreachable!()
};
assert_eq!("local-hdfs", hdfs.name);
assert_eq!("/greptimedb", hdfs.connection.root);
assert_eq!("hdfs://127.0.0.1:9000", hdfs.connection.name_node);
assert_eq!(
Some(&"true".to_string()),
hdfs.connection
.options
.get("dfs.client.block.write.replace-datanode-on-failure.enable")
);
assert!(!hdfs.cache.enable_read_cache);
}
#[allow(deprecated)]
#[test]
fn test_load_datanode_example_config() {