feat: add HDFS object storage backend (#8701)

* feat: add HDFS object storage backend

Signed-off-by: Minghan2005 <cambrianocean@gmail.com>

* fix: make HDFS storage operations durable

Gate the native HDFS backend behind an explicit feature. Publish writes through same-directory temporary files and atomic HDFS Rename2 replacement, and provide streaming copy fallback for COPY_REGION. Add regression coverage for interrupted writes and the region-copy path.

Signed-off-by: Minghan2005 <cambrianocean@gmail.com>

* ci: run HDFS object store tests

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

* docs: note HDFS temporary file cleanup follow-up

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

* feat: enable HDFS object storage by default

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

* docs: remove redundant HDFS build feature notes

Signed-off-by: jeremyhi <fengjiachun@gmail.com>

---------

Signed-off-by: Minghan2005 <cambrianocean@gmail.com>
Signed-off-by: jeremyhi <fengjiachun@gmail.com>
Co-authored-by: Minghan2005 <cambrianocean@gmail.com>
Co-authored-by: jeremyhi <fengjiachun@gmail.com>

src/cmd/Cargo.toml

@@ -22,6 +22,7 @@ required-features = ["dev-tools"]
 [features]
 default = [
     "ai_functions",
+    "hdfs-object-store",
     "servers/pprof",
     "servers/mem-prof",
     "meta-srv/pg_kvbackend",
@@ -32,6 +33,7 @@ enterprise = ["common-meta/enterprise", "frontend/enterprise", "meta-srv/enterpr
 # Developer-only helper binaries and diagnostic datanode commands.
 # Kept out of `default` so normal/release builds don't compile them.
 dev-tools = []
+hdfs-object-store = ["mito2/hdfs-object-store", "object-store/hdfs-object-store"]
 mysql-object-store = ["object-store/mysql-object-store"]
 tokio-console = ["common-telemetry/tokio-console"]