diff --git a/Cargo.lock b/Cargo.lock
index 17d5882e5e..b59e53b371 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -12671,7 +12671,7 @@ dependencies = [
[[package]]
name = "rskafka"
version = "0.6.0"
-source = "git+https://github.com/GreptimeTeam/rskafka.git?rev=f5688f83e7da591cda3f2674c2408b4c0ed4ed50#f5688f83e7da591cda3f2674c2408b4c0ed4ed50"
+source = "git+https://github.com/GreptimeTeam/rskafka.git?rev=5f9494104f7b43a619d6c42e2a2082e39a7b72aa#5f9494104f7b43a619d6c42e2a2082e39a7b72aa"
dependencies = [
"bytes",
"chrono",
diff --git a/Cargo.toml b/Cargo.toml
index 642352674d..150880813c 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -223,7 +223,7 @@ reqwest = { version = "0.13", default-features = false, features = [
url = "2.3"
# Branch: feat/request-timeout
hostname = "0.4.0"
-rskafka = { git = "https://github.com/GreptimeTeam/rskafka.git", rev = "f5688f83e7da591cda3f2674c2408b4c0ed4ed50", features = [
+rskafka = { git = "https://github.com/GreptimeTeam/rskafka.git", rev = "5f9494104f7b43a619d6c42e2a2082e39a7b72aa", features = [
"transport-tls",
] }
rstest = "0.25"
diff --git a/config/config.md b/config/config.md
index 319110809d..8dcfacb1b8 100644
--- a/config/config.md
+++ b/config/config.md
@@ -106,7 +106,7 @@
| `wal.recovery_parallelism` | Integer | `2` | Parallelism during WAL recovery. |
| `wal.broker_endpoints` | Array | -- | The Kafka broker endpoints.
**It's only used when the provider is `kafka`**. |
| `wal.connect_timeout` | String | `3s` | The connect timeout for kafka client.
**It's only used when the provider is `kafka`**. |
-| `wal.timeout` | String | `3s` | The timeout for kafka client.
**It's only used when the provider is `kafka`**. |
+| `wal.timeout` | String | `5s` | The total request timeout for kafka client.
**It's only used when the provider is `kafka`**. |
| `wal.auto_create_topics` | Bool | `true` | Automatically create topics for WAL.
Set to `true` to automatically create topics for WAL.
Otherwise, use topics named `topic_name_prefix_[0..num_topics)` |
| `wal.num_topics` | Integer | `64` | Number of topics.
**It's only used when the provider is `kafka`**. |
| `wal.selector_type` | String | `round_robin` | Topic selector type.
Available selector types:
- `round_robin` (default)
**It's only used when the provider is `kafka`**. |
@@ -460,6 +460,8 @@
| `wal` | -- | -- | -- |
| `wal.provider` | String | `raft_engine` | -- |
| `wal.broker_endpoints` | Array | -- | The broker endpoints of the Kafka cluster.
**It's only used when the provider is `kafka`**. |
+| `wal.connect_timeout` | String | `3s` | The connect timeout for kafka client.
**It's only used when the provider is `kafka`**. |
+| `wal.timeout` | String | `5s` | The total request timeout for kafka client.
**It's only used when the provider is `kafka`**. |
| `wal.auto_create_topics` | Bool | `true` | Automatically create topics for WAL.
Set to `true` to automatically create topics for WAL.
Otherwise, use topics named `topic_name_prefix_[0..num_topics)`
**It's only used when the provider is `kafka`**. |
| `wal.auto_prune_interval` | String | `30m` | Interval of automatically WAL pruning.
Set to `0s` to disable automatically WAL pruning which delete unused remote WAL entries periodically.
**It's only used when the provider is `kafka`**. |
| `wal.auto_prune_logical_delete` | Bool | `false` | Whether automatically WAL pruning only updates the metadata marker and skips Kafka DeleteRecords.
Set to `true` for Kafka deployments that do not support DeleteRecords.
**It's only used when the provider is `kafka`**. |
@@ -561,7 +563,7 @@
| `wal.recovery_parallelism` | Integer | `2` | Parallelism during WAL recovery. |
| `wal.broker_endpoints` | Array | -- | The Kafka broker endpoints.
**It's only used when the provider is `kafka`**. |
| `wal.connect_timeout` | String | `3s` | The connect timeout for kafka client.
**It's only used when the provider is `kafka`**. |
-| `wal.timeout` | String | `3s` | The timeout for kafka client.
**It's only used when the provider is `kafka`**. |
+| `wal.timeout` | String | `5s` | The total request timeout for kafka client.
**It's only used when the provider is `kafka`**. |
| `wal.max_batch_bytes` | String | `1MB` | The max size of a single producer batch.
Warning: Kafka has a default limit of 1MB per message in a topic.
**It's only used when the provider is `kafka`**. |
| `wal.consumer_wait_timeout` | String | `100ms` | The consumer wait timeout.
**It's only used when the provider is `kafka`**. |
| `wal.create_index` | Bool | `false` | Whether to enable WAL index creation.
**It's only used when the provider is `kafka`**. |
diff --git a/config/datanode.example.toml b/config/datanode.example.toml
index a99c817ae2..decc69dbf5 100644
--- a/config/datanode.example.toml
+++ b/config/datanode.example.toml
@@ -186,9 +186,9 @@ broker_endpoints = ["127.0.0.1:9092"]
## **It's only used when the provider is `kafka`**.
#+ connect_timeout = "3s"
-## The timeout for kafka client.
+## The total request timeout for kafka client.
## **It's only used when the provider is `kafka`**.
-#+ timeout = "3s"
+#+ timeout = "5s"
## The max size of a single producer batch.
## Warning: Kafka has a default limit of 1MB per message in a topic.
diff --git a/config/metasrv.example.toml b/config/metasrv.example.toml
index ae373d0748..5ecbc924f7 100644
--- a/config/metasrv.example.toml
+++ b/config/metasrv.example.toml
@@ -229,6 +229,14 @@ provider = "raft_engine"
## **It's only used when the provider is `kafka`**.
broker_endpoints = ["127.0.0.1:9092"]
+## The connect timeout for kafka client.
+## **It's only used when the provider is `kafka`**.
+#+ connect_timeout = "3s"
+
+## The total request timeout for kafka client.
+## **It's only used when the provider is `kafka`**.
+#+ timeout = "5s"
+
## Automatically create topics for WAL.
## Set to `true` to automatically create topics for WAL.
## Otherwise, use topics named `topic_name_prefix_[0..num_topics)`
diff --git a/config/standalone.example.toml b/config/standalone.example.toml
index 0444863727..696d739781 100644
--- a/config/standalone.example.toml
+++ b/config/standalone.example.toml
@@ -305,9 +305,9 @@ broker_endpoints = ["127.0.0.1:9092"]
## **It's only used when the provider is `kafka`**.
#+ connect_timeout = "3s"
-## The timeout for kafka client.
+## The total request timeout for kafka client.
## **It's only used when the provider is `kafka`**.
-#+ timeout = "3s"
+#+ timeout = "5s"
## Automatically create topics for WAL.
## Set to `true` to automatically create topics for WAL.
diff --git a/src/common/wal/Cargo.toml b/src/common/wal/Cargo.toml
index b9ca39efca..62fdee79ae 100644
--- a/src/common/wal/Cargo.toml
+++ b/src/common/wal/Cargo.toml
@@ -28,5 +28,6 @@ snafu.workspace = true
tokio.workspace = true
[dev-dependencies]
+rskafka = { workspace = true, features = ["unstable-fuzzing"] }
serde_json.workspace = true
toml.workspace = true
diff --git a/src/common/wal/src/config.rs b/src/common/wal/src/config.rs
index 5a0d5db426..f00d615301 100644
--- a/src/common/wal/src/config.rs
+++ b/src/common/wal/src/config.rs
@@ -214,7 +214,7 @@ mod tests {
client_key_path: None,
}),
connect_timeout: Duration::from_secs(3),
- timeout: Duration::from_secs(3),
+ timeout: Duration::from_secs(5),
},
kafka_topic: KafkaTopicConfig {
num_topics: 32,
@@ -252,7 +252,7 @@ mod tests {
client_key_path: None,
}),
connect_timeout: Duration::from_secs(3),
- timeout: Duration::from_secs(3),
+ timeout: Duration::from_secs(5),
},
max_batch_bytes: ReadableSize::mb(1),
consumer_wait_timeout: Duration::from_millis(100),
@@ -291,3 +291,33 @@ mod tests {
assert!(!debug.contains("kafka-secret"));
}
}
+
+#[cfg(test)]
+mod rskafka_contract_tests {
+ use std::sync::Arc;
+ use std::time::Duration;
+
+ use rskafka::build_info::DEFAULT_CLIENT_ID;
+ use rskafka::messenger::{Messenger, RequestError, SyncVersionsError};
+
+ #[tokio::test]
+ async fn test_send_timeout_poisoned_messenger() {
+ let (_peer, stream) = tokio::io::duplex(1);
+ let mut messenger = Messenger::new(
+ stream,
+ 1_000,
+ Arc::from(DEFAULT_CLIENT_ID),
+ Some(Duration::from_millis(50)),
+ );
+
+ let error = messenger.sync_versions().await.unwrap_err();
+ assert!(matches!(
+ error,
+ SyncVersionsError::RequestError(RequestError::Poisoned(error))
+ if matches!(
+ error.as_ref(),
+ RequestError::IO(error) if error.kind() == std::io::ErrorKind::TimedOut
+ )
+ ));
+ }
+}
diff --git a/src/common/wal/src/config/kafka/common.rs b/src/common/wal/src/config/kafka/common.rs
index 4266f30a85..cf632fbdce 100644
--- a/src/common/wal/src/config/kafka/common.rs
+++ b/src/common/wal/src/config/kafka/common.rs
@@ -198,7 +198,7 @@ pub struct KafkaConnectionConfig {
/// The connect timeout for kafka client.
#[serde(with = "humantime_serde")]
pub connect_timeout: Duration,
- /// The timeout for kafka client.
+ /// The total request timeout for kafka client.
#[serde(with = "humantime_serde")]
pub timeout: Duration,
}
@@ -210,7 +210,7 @@ impl Default for KafkaConnectionConfig {
sasl: None,
tls: None,
connect_timeout: Duration::from_secs(3),
- timeout: Duration::from_secs(3),
+ timeout: Duration::from_secs(5),
}
}
}
diff --git a/src/log-store/src/metrics.rs b/src/log-store/src/metrics.rs
index c82be98f66..f12ad3e815 100644
--- a/src/log-store/src/metrics.rs
+++ b/src/log-store/src/metrics.rs
@@ -54,6 +54,10 @@ lazy_static! {
"greptime_logstore_op_elapsed",
"logstore operation elapsed",
&[LOGSTORE_LABEL, OPTYPE_LABEL],
+ vec![
+ 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0,
+ 45.0, 60.0,
+ ],
)
.unwrap();
/// Timer of the append_batch operation on the kafka logstore.