diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml
index 9a3ff5e845..f01f8c145c 100644
--- a/.github/workflows/develop.yml
+++ b/.github/workflows/develop.yml
@@ -763,6 +763,7 @@ jobs:
GT_MINIO_ACCESS_KEY: superpower_password
GT_MINIO_REGION: us-west-2
GT_MINIO_ENDPOINT_URL: http://127.0.0.1:9000
+ GT_ETCD_TLS_ENDPOINTS: https://127.0.0.1:2378
GT_ETCD_ENDPOINTS: http://127.0.0.1:2379
GT_POSTGRES_ENDPOINTS: postgres://greptimedb:admin@127.0.0.1:5432/postgres
GT_MYSQL_ENDPOINTS: mysql://greptimedb:admin@127.0.0.1:3306/mysql
@@ -815,6 +816,7 @@ jobs:
GT_MINIO_ACCESS_KEY: superpower_password
GT_MINIO_REGION: us-west-2
GT_MINIO_ENDPOINT_URL: http://127.0.0.1:9000
+ GT_ETCD_TLS_ENDPOINTS: https://127.0.0.1:2378
GT_ETCD_ENDPOINTS: http://127.0.0.1:2379
GT_POSTGRES_ENDPOINTS: postgres://greptimedb:admin@127.0.0.1:5432/postgres
GT_MYSQL_ENDPOINTS: mysql://greptimedb:admin@127.0.0.1:3306/mysql
diff --git a/Cargo.lock b/Cargo.lock
index c86a7f77a2..37b8bb7736 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -13486,6 +13486,7 @@ dependencies = [
"percent-encoding",
"pin-project",
"prost 0.13.5",
+ "rustls-native-certs 0.8.1",
"socket2 0.5.10",
"tokio",
"tokio-rustls",
diff --git a/Cargo.toml b/Cargo.toml
index a2e8d368e4..47748c4e40 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -138,7 +138,10 @@ deadpool-postgres = "0.14"
derive_builder = "0.20"
dotenv = "0.15"
either = "1.15"
-etcd-client = { git = "https://github.com/GreptimeTeam/etcd-client", rev = "f62df834f0cffda355eba96691fe1a9a332b75a7" }
+etcd-client = { git = "https://github.com/GreptimeTeam/etcd-client", rev = "f62df834f0cffda355eba96691fe1a9a332b75a7", features = [
+ "tls",
+ "tls-roots",
+] }
fst = "0.4.7"
futures = "0.3"
futures-util = "0.3"
diff --git a/config/config.md b/config/config.md
index 7b3f4745b3..de44c6de89 100644
--- a/config/config.md
+++ b/config/config.md
@@ -344,7 +344,7 @@
| `runtime` | -- | -- | The runtime options. |
| `runtime.global_rt_size` | Integer | `8` | The number of threads to execute the runtime for global read operations. |
| `runtime.compact_rt_size` | Integer | `4` | The number of threads to execute the runtime for global write operations. |
-| `backend_tls` | -- | -- | TLS configuration for kv store backend (only applicable for PostgreSQL/MySQL backends)
When using PostgreSQL or MySQL as metadata store, you can configure TLS here |
+| `backend_tls` | -- | -- | TLS configuration for kv store backend (applicable for etcd, PostgreSQL, and MySQL backends)
When using etcd, PostgreSQL, or MySQL as metadata store, you can configure TLS here |
| `backend_tls.mode` | String | `prefer` | TLS mode, refer to https://www.postgresql.org/docs/current/libpq-ssl.html
- "disable" - No TLS
- "prefer" (default) - Try TLS, fallback to plain
- "require" - Require TLS
- "verify_ca" - Require TLS and verify CA
- "verify_full" - Require TLS and verify hostname |
| `backend_tls.cert_path` | String | `""` | Path to client certificate file (for client authentication)
Like "/path/to/client.crt" |
| `backend_tls.key_path` | String | `""` | Path to client private key file (for client authentication)
Like "/path/to/client.key" |
diff --git a/config/metasrv.example.toml b/config/metasrv.example.toml
index a7f53eea49..9dc629e76f 100644
--- a/config/metasrv.example.toml
+++ b/config/metasrv.example.toml
@@ -65,8 +65,8 @@ node_max_idle_time = "24hours"
## The number of threads to execute the runtime for global write operations.
#+ compact_rt_size = 4
-## TLS configuration for kv store backend (only applicable for PostgreSQL/MySQL backends)
-## When using PostgreSQL or MySQL as metadata store, you can configure TLS here
+## TLS configuration for kv store backend (applicable for etcd, PostgreSQL, and MySQL backends)
+## When using etcd, PostgreSQL, or MySQL as metadata store, you can configure TLS here
[backend_tls]
## TLS mode, refer to https://www.postgresql.org/docs/current/libpq-ssl.html
## - "disable" - No TLS
diff --git a/docker/docker-compose/cluster-with-etcd.yaml b/docker/docker-compose/cluster-with-etcd.yaml
index 4ad90f1285..34eee7f4b5 100644
--- a/docker/docker-compose/cluster-with-etcd.yaml
+++ b/docker/docker-compose/cluster-with-etcd.yaml
@@ -34,6 +34,48 @@ services:
networks:
- greptimedb
+ etcd-tls:
+ <<: *etcd_common_settings
+ container_name: etcd-tls
+ ports:
+ - 2378:2378
+ - 2381:2381
+ command:
+ - --name=etcd-tls
+ - --data-dir=/var/lib/etcd
+ - --initial-advertise-peer-urls=https://etcd-tls:2381
+ - --listen-peer-urls=https://0.0.0.0:2381
+ - --listen-client-urls=https://0.0.0.0:2378
+ - --advertise-client-urls=https://etcd-tls:2378
+ - --heartbeat-interval=250
+ - --election-timeout=1250
+ - --initial-cluster=etcd-tls=https://etcd-tls:2381
+ - --initial-cluster-state=new
+ - --initial-cluster-token=etcd-tls-cluster
+ - --cert-file=/certs/server.crt
+ - --key-file=/certs/server-key.pem
+ - --peer-cert-file=/certs/server.crt
+ - --peer-key-file=/certs/server-key.pem
+ - --trusted-ca-file=/certs/ca.crt
+ - --peer-trusted-ca-file=/certs/ca.crt
+ - --client-cert-auth
+ - --peer-client-cert-auth
+ volumes:
+ - ./greptimedb-cluster-docker-compose/etcd-tls:/var/lib/etcd
+ - ./greptimedb-cluster-docker-compose/certs:/certs:ro
+ environment:
+ - ETCDCTL_API=3
+ - ETCDCTL_CACERT=/certs/ca.crt
+ - ETCDCTL_CERT=/certs/server.crt
+ - ETCDCTL_KEY=/certs/server-key.pem
+ healthcheck:
+ test: [ "CMD", "etcdctl", "--endpoints=https://etcd-tls:2378", "--cacert=/certs/ca.crt", "--cert=/certs/server.crt", "--key=/certs/server-key.pem", "endpoint", "health" ]
+ interval: 10s
+ timeout: 5s
+ retries: 5
+ networks:
+ - greptimedb
+
metasrv:
image: *greptimedb_image
container_name: metasrv
diff --git a/scripts/generate-etcd-tls-certs.sh b/scripts/generate-etcd-tls-certs.sh
new file mode 100755
index 0000000000..c7a815d263
--- /dev/null
+++ b/scripts/generate-etcd-tls-certs.sh
@@ -0,0 +1,71 @@
+#!/bin/bash
+
+# Generate TLS certificates for etcd testing
+# This script creates certificates for TLS-enabled etcd in testing environments
+
+set -euo pipefail
+
+CERT_DIR="${1:-$(dirname "$0")/../tests-integration/fixtures/etcd-tls-certs}"
+DAYS="${2:-365}"
+
+echo "Generating TLS certificates for etcd in ${CERT_DIR}..."
+
+mkdir -p "${CERT_DIR}"
+cd "${CERT_DIR}"
+
+echo "Generating CA private key..."
+openssl genrsa -out ca-key.pem 2048
+
+echo "Generating CA certificate..."
+openssl req -new -x509 -key ca-key.pem -out ca.crt -days "${DAYS}" \
+ -subj "/C=US/ST=CA/L=SF/O=Greptime/CN=etcd-ca"
+
+# Create server certificate config with Subject Alternative Names
+echo "Creating server certificate configuration..."
+cat > server.conf << 'EOF'
+[req]
+distinguished_name = req
+[v3_req]
+basicConstraints = CA:FALSE
+keyUsage = keyEncipherment, dataEncipherment
+subjectAltName = @alt_names
+[alt_names]
+DNS.1 = localhost
+DNS.2 = etcd-tls
+DNS.3 = 127.0.0.1
+IP.1 = 127.0.0.1
+IP.2 = ::1
+EOF
+
+echo "Generating server private key..."
+openssl genrsa -out server-key.pem 2048
+
+echo "Generating server certificate signing request..."
+openssl req -new -key server-key.pem -out server.csr \
+ -subj "/CN=etcd-tls"
+
+echo "Generating server certificate..."
+openssl x509 -req -in server.csr -CA ca.crt \
+ -CAkey ca-key.pem -CAcreateserial -out server.crt \
+ -days "${DAYS}" -extensions v3_req -extfile server.conf
+
+echo "Generating client private key..."
+openssl genrsa -out client-key.pem 2048
+
+echo "Generating client certificate signing request..."
+openssl req -new -key client-key.pem -out client.csr \
+ -subj "/CN=etcd-client"
+
+echo "Generating client certificate..."
+openssl x509 -req -in client.csr -CA ca.crt \
+ -CAkey ca-key.pem -CAcreateserial -out client.crt \
+ -days "${DAYS}"
+
+echo "Setting proper file permissions..."
+chmod 644 ca.crt server.crt client.crt
+chmod 600 ca-key.pem server-key.pem client-key.pem
+
+# Clean up intermediate files
+rm -f server.csr client.csr server.conf
+
+echo "TLS certificates generated successfully in ${CERT_DIR}"
diff --git a/src/cli/src/metadata/common.rs b/src/cli/src/metadata/common.rs
index 2455aa400c..960b13f065 100644
--- a/src/cli/src/metadata/common.rs
+++ b/src/cli/src/metadata/common.rs
@@ -19,8 +19,9 @@ use common_error::ext::BoxedError;
use common_meta::kv_backend::chroot::ChrootKvBackend;
use common_meta::kv_backend::etcd::EtcdStore;
use common_meta::kv_backend::KvBackendRef;
-use meta_srv::bootstrap::create_etcd_client;
+use meta_srv::bootstrap::create_etcd_client_with_tls;
use meta_srv::metasrv::BackendImpl;
+use servers::tls::{TlsMode, TlsOption};
use crate::error::{EmptyStoreAddrsSnafu, UnsupportedMemoryBackendSnafu};
@@ -55,6 +56,26 @@ pub(crate) struct StoreConfig {
#[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
#[clap(long, default_value = common_meta::kv_backend::DEFAULT_META_TABLE_NAME)]
meta_table_name: String,
+
+ /// TLS mode for backend store connections (etcd, PostgreSQL, MySQL)
+ #[clap(long = "backend-tls-mode", value_enum, default_value = "disable")]
+ backend_tls_mode: TlsMode,
+
+ /// Path to TLS certificate file for backend store connections
+ #[clap(long = "backend-tls-cert-path", default_value = "")]
+ backend_tls_cert_path: String,
+
+ /// Path to TLS private key file for backend store connections
+ #[clap(long = "backend-tls-key-path", default_value = "")]
+ backend_tls_key_path: String,
+
+ /// Path to TLS CA certificate file for backend store connections
+ #[clap(long = "backend-tls-ca-cert-path", default_value = "")]
+ backend_tls_ca_cert_path: String,
+
+ /// Enable watching TLS certificate files for changes
+ #[clap(long = "backend-tls-watch")]
+ backend_tls_watch: bool,
}
impl StoreConfig {
@@ -67,7 +88,18 @@ impl StoreConfig {
} else {
let kvbackend = match self.backend {
BackendImpl::EtcdStore => {
- let etcd_client = create_etcd_client(store_addrs)
+ let tls_config = if self.backend_tls_mode != TlsMode::Disable {
+ Some(TlsOption {
+ mode: self.backend_tls_mode.clone(),
+ cert_path: self.backend_tls_cert_path.clone(),
+ key_path: self.backend_tls_key_path.clone(),
+ ca_cert_path: self.backend_tls_ca_cert_path.clone(),
+ watch: self.backend_tls_watch,
+ })
+ } else {
+ None
+ };
+ let etcd_client = create_etcd_client_with_tls(store_addrs, tls_config.as_ref())
.await
.map_err(BoxedError::new)?;
Ok(EtcdStore::with_etcd_client(etcd_client, max_txn_ops))
diff --git a/src/meta-srv/src/bootstrap.rs b/src/meta-srv/src/bootstrap.rs
index d77ec335c1..86c917051d 100644
--- a/src/meta-srv/src/bootstrap.rs
+++ b/src/meta-srv/src/bootstrap.rs
@@ -21,6 +21,7 @@ use api::v1::meta::procedure_service_server::ProcedureServiceServer;
use api::v1::meta::store_server::StoreServer;
use common_base::Plugins;
use common_config::Configurable;
+#[cfg(feature = "pg_kvbackend")]
use common_error::ext::BoxedError;
#[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
use common_meta::distributed_time_constants::META_LEASE_SECS;
@@ -40,7 +41,7 @@ use common_telemetry::info;
#[cfg(feature = "pg_kvbackend")]
use deadpool_postgres::{Config, Runtime};
use either::Either;
-use etcd_client::Client;
+use etcd_client::{Client, ConnectOptions};
use servers::configurator::ConfiguratorRef;
use servers::export_metrics::ExportMetricsTask;
use servers::http::{HttpServer, HttpServerBuilder};
@@ -286,7 +287,8 @@ pub async fn metasrv_builder(
(Some(kv_backend), _) => (kv_backend, None),
(None, BackendImpl::MemoryStore) => (Arc::new(MemoryKvBackend::new()) as _, None),
(None, BackendImpl::EtcdStore) => {
- let etcd_client = create_etcd_client(&opts.store_addrs).await?;
+ let etcd_client =
+ create_etcd_client_with_tls(&opts.store_addrs, opts.backend_tls.as_ref()).await?;
let kv_backend = EtcdStore::with_etcd_client(etcd_client.clone(), opts.max_txn_ops);
let election = EtcdElection::with_etcd_client(
&opts.grpc.server_addr,
@@ -435,12 +437,67 @@ pub async fn metasrv_builder(
}
pub async fn create_etcd_client(store_addrs: &[String]) -> Result {
+ create_etcd_client_with_tls(store_addrs, None).await
+}
+
+fn build_connection_options(tls_config: Option<&TlsOption>) -> Result