1use std::collections::{HashMap, HashSet};
16use std::path::Path;
17use std::sync::{Mutex, OnceLock};
18
19use serde::Serialize;
20use tinytemplate::TinyTemplate;
21
22use crate::cmd::bare::ServerAddr;
23use crate::cmd::compat_case::Version;
24use crate::env::bare::{Env, GreptimeDBContext, ServiceProvider};
25use crate::util;
26
27const DEFAULT_LOG_LEVEL: &str = "--log-level=debug,hyper=warn,tower=warn,datafusion=warn,reqwest=warn,sqlparser=warn,h2=info,opendal=info";
28
29#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
36pub enum GrpcArgStyle {
37 Grpc,
39 Rpc,
41}
42
43impl GrpcArgStyle {
44 pub(crate) fn for_version(version: Option<&Version>) -> Self {
49 const GRPC_ARG_RENAME_VERSION: Version = Version {
50 major: 1,
51 minor: 1,
52 patch: 0,
53 };
54
55 if version.is_some_and(|version| version < &GRPC_ARG_RENAME_VERSION) {
56 GrpcArgStyle::Rpc
57 } else {
58 GrpcArgStyle::Grpc
59 }
60 }
61
62 pub fn bind_addr_arg(self) -> &'static str {
64 match self {
65 GrpcArgStyle::Grpc => "--grpc-bind-addr",
66 GrpcArgStyle::Rpc => "--rpc-bind-addr",
67 }
68 }
69
70 pub fn server_addr_arg(self) -> &'static str {
72 match self {
73 GrpcArgStyle::Grpc => "--grpc-server-addr",
74 GrpcArgStyle::Rpc => "--rpc-server-addr",
75 }
76 }
77}
78
79static USED_PORTS: OnceLock<Mutex<HashSet<u16>>> = OnceLock::new();
80
81fn get_used_ports() -> &'static Mutex<HashSet<u16>> {
82 USED_PORTS.get_or_init(|| Mutex::new(HashSet::new()))
83}
84
85fn get_unique_random_port() -> u16 {
86 const MAX_ATTEMPTS: usize = 100;
88
89 for _ in 0..MAX_ATTEMPTS {
90 let p = util::get_random_port();
91 let mut used = get_used_ports().lock().unwrap();
92 if !used.contains(&p) {
93 used.insert(p);
94 return p;
95 }
96 }
97
98 panic!(
99 "Failed to find an unused port after {} attempts",
100 MAX_ATTEMPTS
101 );
102}
103
104#[derive(Clone)]
105pub enum ServerMode {
106 Standalone {
107 http_addr: String,
108 rpc_bind_addr: String,
109 mysql_addr: String,
110 postgres_addr: String,
111 },
112 Frontend {
113 http_addr: String,
114 rpc_bind_addr: String,
115 mysql_addr: String,
116 postgres_addr: String,
117 metasrv_addr: String,
118 },
119 Metasrv {
120 rpc_bind_addr: String,
121 rpc_server_addr: String,
122 http_addr: String,
123 },
124 Datanode {
125 rpc_bind_addr: String,
126 rpc_server_addr: String,
127 http_addr: String,
128 metasrv_addr: String,
129 node_id: u32,
130 },
131 Flownode {
132 rpc_bind_addr: String,
133 rpc_server_addr: String,
134 http_addr: String,
135 metasrv_addr: String,
136 node_id: u32,
137 },
138}
139
140#[derive(Serialize)]
141struct ConfigContext {
142 wal_dir: String,
143 data_home: String,
144 procedure_dir: String,
145 is_raft_engine: bool,
146 kafka_wal_broker_endpoints: String,
147 use_etcd: bool,
148 store_addrs: String,
149 instance_id: usize,
150 addrs: HashMap<String, String>,
151 enable_flat_format: bool,
153}
154
155impl ServerMode {
156 pub fn random_standalone() -> Self {
157 let http_port = get_unique_random_port();
158 let rpc_port = get_unique_random_port();
159 let mysql_port = get_unique_random_port();
160 let postgres_port = get_unique_random_port();
161
162 ServerMode::Standalone {
163 http_addr: format!("127.0.0.1:{http_port}"),
164 rpc_bind_addr: format!("127.0.0.1:{rpc_port}"),
165 mysql_addr: format!("127.0.0.1:{mysql_port}"),
166 postgres_addr: format!("127.0.0.1:{postgres_port}"),
167 }
168 }
169
170 pub fn random_frontend(metasrv_port: u16) -> Self {
171 let http_port = get_unique_random_port();
172 let rpc_port = get_unique_random_port();
173 let mysql_port = get_unique_random_port();
174 let postgres_port = get_unique_random_port();
175
176 ServerMode::Frontend {
177 http_addr: format!("127.0.0.1:{http_port}"),
178 rpc_bind_addr: format!("127.0.0.1:{rpc_port}"),
179 mysql_addr: format!("127.0.0.1:{mysql_port}"),
180 postgres_addr: format!("127.0.0.1:{postgres_port}"),
181 metasrv_addr: format!("127.0.0.1:{metasrv_port}"),
182 }
183 }
184
185 pub fn random_metasrv() -> Self {
186 let bind_port = get_unique_random_port();
187 let http_port = get_unique_random_port();
188
189 ServerMode::Metasrv {
190 rpc_bind_addr: format!("127.0.0.1:{bind_port}"),
191 rpc_server_addr: format!("127.0.0.1:{bind_port}"),
192 http_addr: format!("127.0.0.1:{http_port}"),
193 }
194 }
195
196 pub fn random_datanode(metasrv_port: u16, node_id: u32) -> Self {
197 let rpc_port = get_unique_random_port();
198 let http_port = get_unique_random_port();
199
200 ServerMode::Datanode {
201 rpc_bind_addr: format!("127.0.0.1:{rpc_port}"),
202 rpc_server_addr: format!("127.0.0.1:{rpc_port}"),
203 http_addr: format!("127.0.0.1:{http_port}"),
204 metasrv_addr: format!("127.0.0.1:{metasrv_port}"),
205 node_id,
206 }
207 }
208
209 pub fn random_flownode(metasrv_port: u16, node_id: u32) -> Self {
210 let rpc_port = get_unique_random_port();
211 let http_port = get_unique_random_port();
212
213 ServerMode::Flownode {
214 rpc_bind_addr: format!("127.0.0.1:{rpc_port}"),
215 rpc_server_addr: format!("127.0.0.1:{rpc_port}"),
216 http_addr: format!("127.0.0.1:{http_port}"),
217 metasrv_addr: format!("127.0.0.1:{metasrv_port}"),
218 node_id,
219 }
220 }
221
222 pub fn name(&self) -> &'static str {
223 match self {
224 ServerMode::Standalone { .. } => "standalone",
225 ServerMode::Frontend { .. } => "frontend",
226 ServerMode::Metasrv { .. } => "metasrv",
227 ServerMode::Datanode { .. } => "datanode",
228 ServerMode::Flownode { .. } => "flownode",
229 }
230 }
231
232 pub fn check_addrs(&self) -> Vec<String> {
234 match self {
235 ServerMode::Standalone {
236 rpc_bind_addr,
237 mysql_addr,
238 postgres_addr,
239 http_addr,
240 ..
241 } => {
242 vec![
243 rpc_bind_addr.clone(),
244 mysql_addr.clone(),
245 postgres_addr.clone(),
246 http_addr.clone(),
247 ]
248 }
249 ServerMode::Frontend {
250 rpc_bind_addr,
251 mysql_addr,
252 postgres_addr,
253 ..
254 } => {
255 vec![
256 rpc_bind_addr.clone(),
257 mysql_addr.clone(),
258 postgres_addr.clone(),
259 ]
260 }
261 ServerMode::Metasrv { rpc_bind_addr, .. } => {
262 vec![rpc_bind_addr.clone()]
263 }
264 ServerMode::Datanode { rpc_bind_addr, .. } => {
265 vec![rpc_bind_addr.clone()]
266 }
267 ServerMode::Flownode { rpc_bind_addr, .. } => {
268 vec![rpc_bind_addr.clone()]
269 }
270 }
271 }
272
273 pub fn server_addr(&self) -> Option<ServerAddr> {
275 match self {
276 ServerMode::Standalone {
277 rpc_bind_addr,
278 mysql_addr,
279 postgres_addr,
280 ..
281 } => Some(ServerAddr {
282 server_addr: Some(rpc_bind_addr.clone()),
283 pg_server_addr: Some(postgres_addr.clone()),
284 mysql_server_addr: Some(mysql_addr.clone()),
285 }),
286 ServerMode::Frontend {
287 rpc_bind_addr,
288 mysql_addr,
289 postgres_addr,
290 ..
291 } => Some(ServerAddr {
292 server_addr: Some(rpc_bind_addr.clone()),
293 pg_server_addr: Some(postgres_addr.clone()),
294 mysql_server_addr: Some(mysql_addr.clone()),
295 }),
296 _ => None,
297 }
298 }
299
300 pub fn generate_config_file(
301 &self,
302 sqlness_home: &Path,
303 db_ctx: &GreptimeDBContext,
304 id: usize,
305 ) -> String {
306 let mut tt = TinyTemplate::new();
307
308 let mut path = util::sqlness_conf_path();
309 path.push(format!("{}-test.toml.template", self.name()));
310 let template = std::fs::read_to_string(&path)
311 .unwrap_or_else(|e| panic!("read file '{}' error: {e}", path.display()));
312 tt.add_template(self.name(), &template).unwrap();
313
314 let data_home = sqlness_home.join(format!("greptimedb-{}-{}", id, self.name()));
315 std::fs::create_dir_all(data_home.as_path()).unwrap();
316
317 let wal_dir = data_home.join("wal").display().to_string();
318 let procedure_dir = data_home.join("procedure").display().to_string();
319
320 let addrs: HashMap<String, String> = match self {
322 ServerMode::Standalone {
323 rpc_bind_addr,
324 mysql_addr,
325 postgres_addr,
326 http_addr,
327 } => [
328 ("http_addr".to_string(), http_addr.clone()),
329 ("grpc_addr".to_string(), rpc_bind_addr.clone()),
330 ("mysql_addr".to_string(), mysql_addr.clone()),
331 ("postgres_addr".to_string(), postgres_addr.clone()),
332 ]
333 .into(),
334 ServerMode::Frontend { rpc_bind_addr, .. } => {
335 [("grpc_addr".to_string(), rpc_bind_addr.clone())].into()
336 }
337 ServerMode::Datanode { metasrv_addr, .. } => {
338 [("metasrv_addr".to_string(), metasrv_addr.clone())].into()
339 }
340 _ => HashMap::new(),
341 };
342
343 let ctx = ConfigContext {
344 wal_dir,
345 data_home: data_home.display().to_string(),
346 procedure_dir,
347 is_raft_engine: db_ctx.is_raft_engine(),
348 kafka_wal_broker_endpoints: db_ctx.kafka_wal_broker_endpoints(),
349 use_etcd: !db_ctx.store_config().store_addrs.is_empty(),
350 store_addrs: db_ctx
351 .store_config()
352 .store_addrs
353 .iter()
354 .map(|p| format!("\"{p}\""))
355 .collect::<Vec<_>>()
356 .join(","),
357 instance_id: id,
358 addrs,
359 enable_flat_format: db_ctx.store_config().enable_flat_format,
360 };
361
362 let rendered = tt.render(self.name(), &ctx).unwrap();
363
364 let conf_file = data_home
365 .join(format!("{}-{}-{}.toml", self.name(), id, db_ctx.time()))
366 .display()
367 .to_string();
368 println!(
369 "Generating id {}, {} config file in {conf_file}",
370 id,
371 self.name()
372 );
373 std::fs::write(&conf_file, rendered).unwrap();
374
375 conf_file
376 }
377
378 pub fn get_args(
379 &self,
380 sqlness_home: &Path,
381 env: &Env,
382 db_ctx: &GreptimeDBContext,
383 id: usize,
384 arg_style: GrpcArgStyle,
385 ) -> Vec<String> {
386 let mut args = env
387 .extra_args()
388 .iter()
389 .map(String::as_str)
390 .chain([DEFAULT_LOG_LEVEL, self.name(), "start"])
391 .map(ToString::to_string)
392 .collect::<Vec<String>>();
393
394 match self {
395 ServerMode::Standalone {
396 http_addr,
397 rpc_bind_addr,
398 mysql_addr,
399 postgres_addr,
400 } => {
401 args.extend([
402 format!(
403 "--log-dir={}/greptimedb-{}-standalone/logs",
404 sqlness_home.display(),
405 id
406 ),
407 "-c".to_string(),
408 self.generate_config_file(sqlness_home, db_ctx, id),
409 format!("--http-addr={http_addr}"),
410 format!("{}={rpc_bind_addr}", arg_style.bind_addr_arg()),
411 format!("--mysql-addr={mysql_addr}"),
412 format!("--postgres-addr={postgres_addr}"),
413 ]);
414 }
415 ServerMode::Frontend {
416 http_addr,
417 rpc_bind_addr,
418 mysql_addr,
419 postgres_addr,
420 metasrv_addr,
421 } => {
422 args.extend([
423 format!("--metasrv-addrs={metasrv_addr}"),
424 format!("--http-addr={http_addr}"),
425 format!("{}={rpc_bind_addr}", arg_style.bind_addr_arg()),
426 format!("{}={rpc_bind_addr}", arg_style.server_addr_arg()),
429 format!("--mysql-addr={mysql_addr}"),
430 format!("--postgres-addr={postgres_addr}"),
431 format!(
432 "--log-dir={}/greptimedb-{}-frontend/logs",
433 sqlness_home.display(),
434 id
435 ),
436 "-c".to_string(),
437 self.generate_config_file(sqlness_home, db_ctx, id),
438 ]);
439 }
440 ServerMode::Metasrv {
441 rpc_bind_addr,
442 rpc_server_addr,
443 http_addr,
444 } => {
445 args.extend([
446 arg_style.bind_addr_arg().to_string(),
447 rpc_bind_addr.clone(),
448 arg_style.server_addr_arg().to_string(),
449 rpc_server_addr.clone(),
450 "--enable-region-failover".to_string(),
451 "false".to_string(),
452 format!("--http-addr={http_addr}"),
453 format!(
454 "--log-dir={}/greptimedb-{}-metasrv/logs",
455 sqlness_home.display(),
456 id
457 ),
458 "-c".to_string(),
459 self.generate_config_file(sqlness_home, db_ctx, id),
460 ]);
461
462 if matches!(
463 db_ctx.store_config().setup_pg,
464 Some(ServiceProvider::Create)
465 ) {
466 let client_ports = db_ctx
467 .store_config()
468 .store_addrs
469 .iter()
470 .map(|s| s.split(':').nth(1).unwrap().parse::<u16>().unwrap())
471 .collect::<Vec<_>>();
472 let client_port = client_ports.first().unwrap_or(&5432);
473 let pg_server_addr = format!(
474 "postgresql://greptimedb:admin@127.0.0.1:{}/postgres",
475 client_port
476 );
477 args.extend(vec!["--backend".to_string(), "postgres-store".to_string()]);
478 args.extend(vec!["--store-addrs".to_string(), pg_server_addr]);
479 } else if let Some(ServiceProvider::External(connection_string)) =
480 db_ctx.store_config().setup_pg
481 {
482 println!("Using external PostgreSQL '{connection_string}' as Kvbackend");
483 args.extend([
484 "--backend".to_string(),
485 "postgres-store".to_string(),
486 "--store-addrs".to_string(),
487 connection_string,
488 ]);
489 } else if matches!(
490 db_ctx.store_config().setup_mysql,
491 Some(ServiceProvider::Create)
492 ) {
493 let client_ports = db_ctx
494 .store_config()
495 .store_addrs
496 .iter()
497 .map(|s| s.split(':').nth(1).unwrap().parse::<u16>().unwrap())
498 .collect::<Vec<_>>();
499 let client_port = client_ports.first().unwrap_or(&3306);
500 let mysql_server_addr =
501 format!("mysql://greptimedb:admin@127.0.0.1:{}/mysql", client_port);
502 args.extend(vec!["--backend".to_string(), "mysql-store".to_string()]);
503 args.extend(vec!["--store-addrs".to_string(), mysql_server_addr]);
504 } else if let Some(ServiceProvider::External(connection_string)) =
505 db_ctx.store_config().setup_mysql
506 {
507 println!("Using external MySQL '{connection_string}' as Kvbackend");
508 args.extend([
509 "--backend".to_string(),
510 "mysql-store".to_string(),
511 "--store-addrs".to_string(),
512 connection_string,
513 ]);
514 } else if db_ctx.store_config().store_addrs.is_empty() {
515 args.extend(vec!["--backend".to_string(), "memory-store".to_string()])
516 }
517 }
518 ServerMode::Datanode {
519 rpc_bind_addr,
520 rpc_server_addr,
521 http_addr,
522 metasrv_addr,
523 node_id,
524 } => {
525 let data_home = sqlness_home.join(format!(
526 "greptimedb_{}_datanode_{}_{node_id}",
527 id,
528 db_ctx.time()
529 ));
530 args.extend([
531 format!("{}={rpc_bind_addr}", arg_style.bind_addr_arg()),
532 format!("{}={rpc_server_addr}", arg_style.server_addr_arg()),
533 format!("--http-addr={http_addr}"),
534 format!("--data-home={}", data_home.display()),
535 format!("--log-dir={}/logs", data_home.display()),
536 format!("--node-id={node_id}"),
537 "-c".to_string(),
538 self.generate_config_file(sqlness_home, db_ctx, id),
539 format!("--metasrv-addrs={metasrv_addr}"),
540 ]);
541 }
542 ServerMode::Flownode {
543 rpc_bind_addr,
544 rpc_server_addr,
545 http_addr,
546 metasrv_addr,
547 node_id,
548 } => {
549 args.extend([
550 format!("{}={rpc_bind_addr}", arg_style.bind_addr_arg()),
551 format!("{}={rpc_server_addr}", arg_style.server_addr_arg()),
552 format!("--node-id={node_id}"),
553 format!(
554 "--log-dir={}/greptimedb-{}-flownode/logs",
555 sqlness_home.display(),
556 id
557 ),
558 format!("--metasrv-addrs={metasrv_addr}"),
559 format!("--http-addr={http_addr}"),
560 ]);
561 }
562 }
563
564 args
565 }
566}
567
568#[cfg(test)]
569mod tests {
570 use std::path::PathBuf;
571
572 use super::*;
573 use crate::env::bare::{StoreConfig, WalConfig};
574
575 fn test_env(sqlness_home: &Path) -> (Env, GreptimeDBContext) {
576 let store_config = StoreConfig {
577 store_addrs: vec!["127.0.0.1:2379".to_string()],
578 setup_etcd: true,
579 setup_pg: None,
580 setup_mysql: None,
581 enable_flat_format: false,
582 };
583 let env = Env::new(
584 sqlness_home.to_path_buf(),
585 ServerAddr::default(),
586 WalConfig::RaftEngine,
587 false,
588 Some(PathBuf::from(".")),
589 store_config.clone(),
590 vec![],
591 );
592 let db_ctx = GreptimeDBContext::new(WalConfig::RaftEngine, store_config);
593
594 (env, db_ctx)
595 }
596
597 fn has_arg(args: &[String], name: &str) -> bool {
598 let prefix = format!("{name}=");
599 args.iter()
600 .any(|arg| arg == name || arg.starts_with(&prefix))
601 }
602
603 fn assert_uses_style(args: &[String], style: GrpcArgStyle, expect_server_addr: bool) {
604 let bind = style.bind_addr_arg();
605 let server = style.server_addr_arg();
606 let other_bind = match style {
607 GrpcArgStyle::Grpc => "--rpc-bind-addr",
608 GrpcArgStyle::Rpc => "--grpc-bind-addr",
609 };
610 let other_server = match style {
611 GrpcArgStyle::Grpc => "--rpc-server-addr",
612 GrpcArgStyle::Rpc => "--grpc-server-addr",
613 };
614
615 assert!(has_arg(args, bind), "missing {bind} in args: {args:?}");
616 assert!(
617 !args.iter().any(|arg| arg.contains(other_bind)),
618 "unexpected {other_bind} in args: {args:?}"
619 );
620
621 if expect_server_addr {
622 assert!(has_arg(args, server), "missing {server} in args: {args:?}");
623 assert!(
624 !args.iter().any(|arg| arg.contains(other_server)),
625 "unexpected {other_server} in args: {args:?}"
626 );
627 }
628 }
629
630 fn test_all_modes(env: &Env, db_ctx: &GreptimeDBContext, temp_dir: &Path, style: GrpcArgStyle) {
631 let standalone = ServerMode::Standalone {
632 http_addr: "127.0.0.1:4000".to_string(),
633 rpc_bind_addr: "127.0.0.1:4001".to_string(),
634 mysql_addr: "127.0.0.1:4002".to_string(),
635 postgres_addr: "127.0.0.1:4003".to_string(),
636 };
637 assert_uses_style(
638 &standalone.get_args(temp_dir, env, db_ctx, 0, style),
639 style,
640 false,
641 );
642
643 let frontend = ServerMode::Frontend {
644 http_addr: "127.0.0.1:4100".to_string(),
645 rpc_bind_addr: "127.0.0.1:4101".to_string(),
646 mysql_addr: "127.0.0.1:4102".to_string(),
647 postgres_addr: "127.0.0.1:4103".to_string(),
648 metasrv_addr: "127.0.0.1:4001".to_string(),
649 };
650 assert_uses_style(
651 &frontend.get_args(temp_dir, env, db_ctx, 0, style),
652 style,
653 true,
654 );
655
656 let metasrv = ServerMode::Metasrv {
657 rpc_bind_addr: "127.0.0.1:4201".to_string(),
658 rpc_server_addr: "127.0.0.1:4201".to_string(),
659 http_addr: "127.0.0.1:4200".to_string(),
660 };
661 assert_uses_style(
662 &metasrv.get_args(temp_dir, env, db_ctx, 0, style),
663 style,
664 true,
665 );
666
667 let datanode = ServerMode::Datanode {
668 rpc_bind_addr: "127.0.0.1:4301".to_string(),
669 rpc_server_addr: "127.0.0.1:4301".to_string(),
670 http_addr: "127.0.0.1:4300".to_string(),
671 metasrv_addr: "127.0.0.1:4001".to_string(),
672 node_id: 0,
673 };
674 assert_uses_style(
675 &datanode.get_args(temp_dir, env, db_ctx, 0, style),
676 style,
677 true,
678 );
679
680 let flownode = ServerMode::Flownode {
681 rpc_bind_addr: "127.0.0.1:4401".to_string(),
682 rpc_server_addr: "127.0.0.1:4401".to_string(),
683 http_addr: "127.0.0.1:4400".to_string(),
684 metasrv_addr: "127.0.0.1:4001".to_string(),
685 node_id: 0,
686 };
687 assert_uses_style(
688 &flownode.get_args(temp_dir, env, db_ctx, 0, style),
689 style,
690 true,
691 );
692 }
693
694 #[test]
695 fn test_get_args_with_grpc_style() {
696 let temp_dir = tempfile::tempdir().unwrap();
697 let (env, db_ctx) = test_env(temp_dir.path());
698 test_all_modes(&env, &db_ctx, temp_dir.path(), GrpcArgStyle::Grpc);
699 }
700
701 #[test]
702 fn test_get_args_with_rpc_style() {
703 let temp_dir = tempfile::tempdir().unwrap();
704 let (env, db_ctx) = test_env(temp_dir.path());
705 test_all_modes(&env, &db_ctx, temp_dir.path(), GrpcArgStyle::Rpc);
706 }
707
708 #[test]
709 fn test_arg_style_for_unknown_version_defaults_to_grpc() {
710 assert_eq!(GrpcArgStyle::for_version(None), GrpcArgStyle::Grpc);
711 }
712
713 #[test]
714 fn test_arg_style_for_legacy_versions_uses_rpc() {
715 let v1_0_0 = Version::parse("v1.0.0").unwrap();
716 let v1_0_9 = Version::parse("v1.0.9").unwrap();
717
718 assert_eq!(GrpcArgStyle::for_version(Some(&v1_0_0)), GrpcArgStyle::Rpc);
719 assert_eq!(GrpcArgStyle::for_version(Some(&v1_0_9)), GrpcArgStyle::Rpc);
720 }
721
722 #[test]
723 fn test_arg_style_for_current_versions_uses_grpc() {
724 let v1_1_0 = Version::parse("v1.1.0").unwrap();
725 let v1_2_0 = Version::parse("v1.2.0").unwrap();
726
727 assert_eq!(GrpcArgStyle::for_version(Some(&v1_1_0)), GrpcArgStyle::Grpc);
728 assert_eq!(GrpcArgStyle::for_version(Some(&v1_2_0)), GrpcArgStyle::Grpc);
729 }
730}